Compare commits

...

2160 Commits

Author SHA1 Message Date
AlphaX-Projects
c184c87c10 Added sync from genesis and reindex 2024-07-04 08:17:53 +02:00
AlphaX-Projects
a02d1cec75
Bump version to 4.5.2 2024-06-30 18:41:04 +02:00
AlphaX-Projects
2d9f1d6d81
Update minimum peer version 2024-06-30 18:38:34 +02:00
crowetic
c1da495dd1
Merge pull request #197 from AlphaX-Projects/master
Bug fixes and additions
2024-06-30 08:48:11 -07:00
AlphaX-Projects
533df9f2b8 Updated dependencies 2024-06-29 15:01:36 +02:00
AlphaX-Projects
44b4b08117 Added block minter thread check 2024-06-29 13:17:07 +02:00
AlphaX-Projects
a140805c36 Updated group transactions (owner check) 2024-06-29 12:58:58 +02:00
AlphaX-Projects
ea1d4dd962 Updated electrumx with non null objects 2024-06-29 12:48:06 +02:00
AlphaX-Projects
a6cbdaafd8 Updated to bouncycastle 1.70 (AT and Core) 2024-06-29 12:23:57 +02:00
AlphaX-Projects
5db808e300 Replaced reference method with qualifier (websockets) 2024-06-29 12:18:52 +02:00
AlphaX-Projects
cc95106019 Removed unnecessary continue (last statement in a loop) 2024-06-29 12:15:55 +02:00
AlphaX-Projects
64537ad705 Fixed synchronization on a non-final field 2024-06-29 12:13:12 +02:00
AlphaX-Projects
806dc6d056 Replaced string.equals() with string.isEmpty() 2024-06-29 12:11:38 +02:00
AlphaX-Projects
d58fbab1b5 Removed pointless boolean Expression 2024-06-29 11:58:25 +02:00
AlphaX-Projects
61ede811cd Replaced size() == / > 0 with isEmpty() 2024-06-29 11:54:41 +02:00
AlphaX-Projects
95d42db773 Removed unnecessary semicolons 2024-06-29 11:52:22 +02:00
AlphaX-Projects
ef07a444d6 Removed unnecessary imports 2024-06-29 11:49:19 +02:00
crowetic
5910ceff80
Merge pull request #194 from kennycud/master
Adding in foreign blockchain server configuration add and remove capa…
2024-05-31 17:44:50 -07:00
f916d3581b Corrected incorrect language in the comments. This was copied and pasted 6 times. 2024-05-27 10:10:45 -07:00
498f409346 Adding in foreign blockchain server configuration add and remove capabilities. Also adding in connection recording and connection setting capabilities. 2024-05-26 11:08:07 -07:00
crowetic
d54f840265
Merge pull request #185 from kennycud/master
Server Info Current Server & Ordering
2024-03-25 09:15:41 -07:00
crowetic
cc740cceec
Merge pull request #187 from AlphaX-Projects/master
Update QDN file management
2024-03-25 09:13:02 -07:00
AlphaX-Projects
df39819de0 Update QDN file management
- Allow files to move around the network more quickly
- Ensuring that metadata isn't saved for blocked names
- More selective disk usage
- Minor fixes
2024-03-25 07:34:26 +01:00
e83b2263f0 some simple error logging 2024-03-23 09:42:33 -07:00
5c90c4bc61 Merge remote-tracking branch 'origin/master' 2024-03-23 06:28:02 -07:00
f7793443f3 call current blockchain height to ensure the current server is set to the blockchain provider and sort the servers to ensure the current server is listed first 2024-03-23 06:25:31 -07:00
crowetic
f6b91df7b6
Merge pull request #181 from kennycud/master
Foreign Coin Trade Fees & Summaries
2024-02-27 12:02:24 -08:00
kennycud
92d589a1ca
Merge branch 'Qortal:master' into master 2024-02-19 04:56:33 -08:00
AlphaX-Projects
c4a7fb3b92 Bump version to 4.5.1 2024-02-18 18:14:33 +01:00
AlphaX-Projects
2d27901f9f Adding new algos 2024-02-18 18:12:43 +01:00
AlphaX-Projects
ce8fb002cc
Merge pull request #179 from lgedgar/log-invalid-timestamps
Add debug logging when invalid timestamp is encountered
2024-02-14 15:41:09 +01:00
3f29116b47 Foreign coin trade transaction summaries 2024-02-12 06:31:17 -08:00
Lance Edgar
d05359dfa9 Add debug logging when invalid timestamp is encountered 2024-02-07 16:52:12 -06:00
kennycud
587b063e6a
Merge branch 'Qortal:master' into master 2024-02-06 15:22:51 -08:00
AlphaX-Projects
9e001dfc16
Remove penalty fix 2024-02-03 19:51:51 +01:00
AlphaX-Projects
55f941467f Remove fix in order for the chain history to remain valid 2024-02-03 14:05:11 +01:00
AlphaX-Projects
4f9a4a2091
Disable fix in order for the chain history to remain valid 2024-02-02 16:40:00 +01:00
crowetic
d579606d2d
Merge pull request #174 from QuickMythril/fix-unit-tests
Update ElectrumX servers & fix unit tests
2024-01-31 19:32:20 -08:00
crowetic
dcedcf8685
Merge pull request #176 from QuickMythril/votes-api-fix
Fixed vote weight results API call
2024-01-31 19:27:01 -08:00
f78764880c
Fixed vote weight results API call
blocksMintedPenalty is a negative value, so it should be added to blocksMinted, not subtracted.
2024-01-31 21:13:19 -05:00
AlphaX-Projects
070f14b3dd
Bump version to 4.5.0 2024-01-30 17:38:34 +01:00
AlphaX-Projects
2120490f4b Fix wrong penaties
This will set incorrectly penalized accounts from previous algo run back to previous state.
2024-01-30 17:36:17 +01:00
6051b85e52 Remove out of service 2024-01-21 08:42:05 -05:00
9c62740f44
Merge branch 'master' into fix-unit-tests 2024-01-21 08:36:41 -05:00
AlphaX-Projects
0ed27228b1
Merge pull request #173 from AlphaX-Projects/master
Out of Service
2024-01-21 14:24:40 +01:00
AlphaX-Projects
b75c2029ac Out of Service 2024-01-21 14:23:46 +01:00
21c45535be Enabled fee updates for foreign coin trade transactions. 2024-01-17 18:19:09 -08:00
AlphaX-Projects
867fe764ca
Bump version to 4.4.2 2024-01-17 17:50:22 +01:00
AlphaX-Projects
140f14f2f4
Set exclude reward share transactions blockheight 2024-01-17 17:49:05 +01:00
AlphaX-Projects
9dd61f0e7a
Merge pull request #170 from AlphaX-Projects/master
Update dependencies
2024-01-17 17:45:42 +01:00
AlphaX-Projects
747b1a4f9d Update dependencies 2024-01-17 17:28:27 +01:00
15ae32efd9 Fix crosschain tests 2024-01-17 06:56:06 -05:00
03ba36729b Update ElectrumX servers 2024-01-17 06:08:09 -05:00
AlphaX-Projects
425152657a
Merge pull request #166 from AlphaX-Projects/master
Exclude reward share transactions from the online accounts blocks
2024-01-14 20:36:46 +01:00
AlphaX-Projects
41645ac7b4 Exclude reward share transactions from the online accounts blocks 2024-01-14 18:40:40 +01:00
AlphaX-Projects
83e324c4ad
Fix file ending 2024-01-07 14:34:20 +01:00
AlphaX-Projects
d7f44376be
Default minPeerVersion set to 4.4.1 2024-01-07 14:32:12 +01:00
AlphaX-Projects
1400e7ae80
Bump version to 4.4.1 2024-01-06 12:25:24 +01:00
AlphaX-Projects
3c116ca4f4
Merge pull request #162 from QuickMythril/testing-20240104
Remove code for unused "Open UI" function
2024-01-06 11:43:16 +01:00
8caf5bf8be Remove code for unused "Open UI" function 2024-01-04 13:08:37 -05:00
AlphaX-Projects
687667c8fe
Update Settings.java 2024-01-04 14:00:33 +01:00
AlphaX-Projects
feb5564666
Merge pull request #161 from crowetic/master
modifications to settings defaults, and start script defaults.
2024-01-04 13:55:52 +01:00
8062cace30 Improve default JVM memory arguments for mac/linux. 2024-01-03 17:23:23 -08:00
bac0f01007 Modified default settings for optimal QDN performance. 2024-01-03 17:06:04 -08:00
AlphaX-Projects
995ed6ab2a
Merge pull request #160 from AlphaX-Projects/master
Rework restart and bootstrap node
2024-01-02 15:04:01 +01:00
AlphaX-Projects
cc02810f0c Update dependencies 2024-01-02 14:26:08 +01:00
AlphaX-Projects
84974775b4 Reduce log spam 2024-01-02 14:17:21 +01:00
AlphaX-Projects
677fd7a64f Rework restart node and bootstrap node 2024-01-01 19:31:22 +01:00
AlphaX-Projects
2d070f343b Update translations 2024-01-01 18:56:09 +01:00
AlphaX-Projects
903fa0346a
Merge pull request #159 from kennycud/master
Added Core API endpoint to repair LTC wallets
2023-12-30 19:31:04 +01:00
be3c6d0b25 Merge remote-tracking branch 'origin/master' 2023-12-28 14:16:22 -08:00
21796341f2 Added Core API endpoint to repair LTC wallets generated before 2024 by moving all the address balances to the first address. 2023-12-28 14:15:57 -08:00
AlphaX-Projects
97b4db4095
Update dependencies 2023-12-28 14:54:20 +01:00
AlphaX-Projects
8977dc7933
Merge pull request #158 from AlphaX-Projects/master
Remove unusedaddress ( no more functional )
2023-12-28 14:38:07 +01:00
AlphaX-Projects
7dbf532616 Remove unusedaddress ( no more functional ) 2023-12-28 14:22:01 +01:00
AlphaX-Projects
d6dec118a9
Merge pull request #157 from kennycud/master
Unused Receive Address Rewrite
2023-12-28 14:03:12 +01:00
AlphaX-Projects
813f16df11
Merge pull request #145 from QuickMythril/api-bootstrap
Added API call for bootstrapping node
2023-12-28 14:02:45 +01:00
aaba6bf4cf Merge remote-tracking branch 'origin/master' 2023-12-22 13:24:16 -08:00
423a1f7bed Practically rewrote the get unused receive address functionality. The prior implementation was skipping addresses. 2023-12-22 13:22:26 -08:00
AlphaX-Projects
e118f4a410
Merge pull request #155 from kennycud/master
Foreign Blockchain Server Configuration Information Offered in the Core API
2023-12-21 18:01:53 +01:00
kennycud
8607c30cb6
Merge branch 'Qortal:master' into master 2023-12-20 04:30:41 -08:00
46a9075faf Extracted Server classes into ChainableServer in order offer foreign blockchain server information to the Core API. They were not extracted into the ServerConfigurationInfo class as stated in a recent commit. 2023-12-20 04:24:51 -08:00
AlphaX-Projects
6b775cc639
Merge pull request #154 from QuickMythril/testing-20231219
Add vote weights to API call
2023-12-20 09:17:13 +01:00
7e509f27fb Merge remote-tracking branch 'origin/master' 2023-12-19 18:34:13 -08:00
de9c3e551d Extracted Server classes into ServerConfigurationInfo in order offer foreign blockchain server information to the Core API 2023-12-19 18:33:36 -08:00
72aa7b7a77 Add vote weights to API call 2023-12-19 20:38:41 -05:00
AlphaX-Projects
7bd570ae71
Merge pull request #150 from AlphaX-Projects/master
Default minPeerVersion set to 4.4.0
2023-12-12 17:20:59 +01:00
AlphaX-Projects
5afa4e5b1a
Default minPeerVersion set to 4.4.0 2023-12-12 17:05:08 +01:00
AlphaX-Projects
3f0999e59d
Merge pull request #149 from AlphaX-Projects/master
Update dependencies
2023-12-12 16:43:24 +01:00
AlphaX-Projects
87b3b037bd
Update dependencies 2023-12-12 16:20:30 +01:00
AlphaX-Projects
3bf54dbd0a
Merge pull request #148 from AlphaX-Projects/master
Update trustless manager and AT 1.4.1
2023-12-12 16:16:34 +01:00
AlphaX-Projects
bf8005aa5a
Update AT 2023-12-12 10:26:30 +01:00
AlphaX-Projects
79b674d2f2
Update AT metadata 2023-12-12 10:25:18 +01:00
AlphaX-Projects
c989e3c413
Rework trustless manager 2023-12-12 10:05:24 +01:00
AlphaX-Projects
f6e398ec0f
Merge pull request #147 from catbref/AT-ref-fix
Chain AT-transaction references to avoid duplicates
2023-12-11 09:03:23 +01:00
catbref
5054773761 Chain AT-transaction references to avoid duplicates 2023-12-10 14:03:47 +00:00
AlphaX-Projects
a2ccf0e7da
Merge pull request #146 from QuickMythril/translate-hebrew
Added Hebrew translations
2023-12-09 09:47:47 +01:00
b266d205b1 Added Hebrew translations 2023-12-08 18:11:02 -05:00
AlphaX-Projects
6cd722e735
Update testchain.json 2023-12-06 16:17:34 +01:00
AlphaX-Projects
1e4d4e178d
Merge pull request #144 from kennycud/master
Added Spendable Attribute to Address Info
2023-12-06 08:55:04 +01:00
deaef4fc4a checking in and pushing the isSpendable and public key generation for testing again 2023-12-03 12:45:31 -08:00
9671c2da61 using public key for address infos 2023-12-03 11:05:16 -08:00
14279e58bc Added API call for bootstrapping node 2023-12-02 23:06:36 -05:00
87a7b9df08 Revert "Ignoring test cases intended for Bitcoiny coins"
This reverts commit e287fa0ebe1996e1d22d094d3a9a7b185aa59008.
2023-11-29 17:51:07 -08:00
3c307f15b3 Merge remote-tracking branch 'origin/master' 2023-11-29 17:30:26 -08:00
e287fa0ebe Ignoring test cases intended for Bitcoiny coins 2023-11-29 17:29:48 -08:00
a94ef17883 Removed API key from admin/summary 2023-11-18 16:44:41 -05:00
94cfcd66cd
Bump version to 4.4.0 2023-11-15 21:57:41 -05:00
dbbe78263d
Merge pull request #143 from AlphaX-Projects/merge-batch-payout
Merge batch payout
2023-11-15 21:48:44 -05:00
AlphaX-Projects
9e8e0df5d6
Set batch payout blockheight 2023-11-15 17:53:25 +01:00
AlphaX-Projects
95948d35ca
Merge pull request #14 from Qortal/master
Merge pull request #142 from AlphaX-Projects/master
2023-11-15 14:06:20 +01:00
9ca2289ed4
Merge pull request #142 from AlphaX-Projects/master
Added 2 Pirate Chain api endpoints
2023-11-10 13:19:13 -05:00
AlphaX-Projects
e86dab066d
Merge pull request #8 from AlphaX-Projects/batch-rewards
Batch rewards
2023-11-09 11:43:14 +01:00
AlphaX-Projects
debb67a676
Merge branch 'merge-batch-payout' into batch-rewards 2023-11-09 11:42:45 +01:00
AlphaX-Projects
6c4ed2496d Added 2 PirateChain api endpoints 2023-11-08 15:29:19 +01:00
AlphaX-Projects
61d6bd9bca
Merge pull request #7 from Qortal/master
Merge master
2023-11-08 12:19:01 +01:00
4e53e6f0ad
Merge pull request #141 from QuickMythril/altcoin-address-infos
Add address infos support for altcoins
2023-11-06 16:17:41 -05:00
183dac5ac4 Add address infos for BTC, DOGE, DGB, RVN 2023-11-06 15:11:55 -05:00
3a723ed116
Revert temporary change 2023-11-06 14:29:35 -05:00
9f2b44484e
Merge pull request #49 from kennycud/master
Add LTC address infos
2023-11-06 14:24:48 -05:00
b05c0158b1
Resolve temporary conflict 2023-11-06 14:23:49 -05:00
d915dc10f3
Default minPeerVersion set to 4.3.2 2023-11-06 13:03:21 -05:00
b0f21c2eee
Merge pull request #140 from AlphaX-Projects/master
Merge arbitary resource cache
2023-11-05 17:07:24 -05:00
AlphaX-Projects
975b0a5f3d
Bump version 4.3.2 2023-11-05 19:15:16 +01:00
0d64e809c7 Ignoring test cases intended for Bitcoiny coins 2023-11-04 15:26:42 -07:00
d4ef175c4d Added Address info support for Litecoin 2023-11-04 14:49:15 -07:00
AlphaX-Projects
3190fed3b9 Optimize imports 2023-11-04 17:11:07 +01:00
AlphaX-Projects
dfcc7bb370 Update UPnP 2023-11-04 16:44:18 +01:00
AlphaX-Projects
8bd9b41a13 Update dependencies 2023-11-04 16:36:59 +01:00
AlphaX-Projects
85300178c7 Update Advance Installer 2023-11-04 16:32:24 +01:00
AlphaX-Projects
e885a69688
Fix status 2023-11-04 16:30:28 +01:00
AlphaX-Projects
050886a496
Merge pull request #1 from AlphaX-Projects/arbitrary-resources-cache
Arbitrary resources cache
2023-11-04 16:21:24 +01:00
AlphaX-Projects
1aab7d8d8f
Merge branch 'master' into arbitrary-resources-cache 2023-11-04 16:20:39 +01:00
5febfaf3ee
Merge pull request #135 from karl-dv/master
Manual review of "nl" translations
2023-10-30 22:03:59 -04:00
9b64f12b7a
Default minPeerVersion set to 4.3.1 2023-10-30 22:01:48 -04:00
karl-dv
6525cac66c Manual review of "nl" translations 2023-10-28 09:11:08 +02:00
033b6adb72
Bump version to 4.3.1 2023-10-19 13:51:53 -04:00
53c3f7899a
Reward share limit activation timestamp set to 1698508800000 (Sat Oct 28 2023 16:00:00 UTC) 2023-10-19 13:48:33 -04:00
f7f2d70b77
Merge pull request #134 from AlphaX-Projects/master
Final dependencies updates and fixes
2023-10-19 12:57:44 -04:00
AlphaX-Projects
5cbd4490cc
Back to Bouncycastle 1.69 2023-10-18 17:29:10 +02:00
AlphaX-Projects
7103f41e36 Reorg pom update dependencies 2023-10-18 13:49:53 +02:00
AlphaX-Projects
2d599ec3c5 Summary of activity past 24 hours 2023-10-17 18:39:17 +02:00
AlphaX-Projects
e40d884c81
Merge pull request #10 from Qortal/master
Merge thread limit
2023-10-17 18:23:59 +02:00
7cbeb00c1a
Merge pull request #132 from Qortal/thread-limits
Added support for thread limits.
2023-10-17 12:10:35 -04:00
AlphaX-Projects
2af8199d9c Update dependencies 2023-10-16 17:27:14 +02:00
AlphaX-Projects
404c5d0300 Filter failed trade after 1 attempt 2023-10-15 17:12:02 +02:00
AlphaX-Projects
db7b17e52e Revert HSQLDB update 2023-10-13 15:57:43 +02:00
AlphaX-Projects
6d202b2b48 Create settings.json 2023-10-12 11:23:36 +02:00
AlphaX-Projects
499e2ac3f4
hide-failed-trade 2023-10-10 23:24:57 +02:00
CalDescent
15eefa4177 Improved batch reward logging and moved it to debug level 2023-10-06 11:03:06 +01:00
AlphaX-Projects
bf270a63ff
Update bouncycastle 2023-09-24 15:19:07 +02:00
CalDescent
9454031b48 Added support for thread limits.
Default thread limits per message type can be specified in Settings.setAdditionalDefaults(), e.g

maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE", 5));

These can also be overridden on a per-node basis in settings.json, e.g

"maxThreadsPerMessageType": [
    { "messageType": "GET_ARBITRARY_DATA_FILE", "limit": 3 },
    { "messageType": "GET_ARBITRARY_DATA_FILE_LIST", "limit": 3 }
]

settings.json values take priority, but any message types that aren't specified in settings.json will still be included from the Settings.java defaults. This allows single message types to be overridden in settings.json without removing the limits for all of the other message types.

Any messages that arrive are discarded if the node is already at the thread limit for that message type.

Warnings are now shown in the logs if the total number of active threads reaches 90% of the allocated thread pool size. Additionally, it can warn per message type by specifying a per-message-type warning threshold in settings.json, e.g

"threadCountPerMessageTypeWarningThreshold": 20

The above setting would warn in the logs if a single message type was consuming more than 20 threads at once, therefore making it a candidate to be limited in maxThreadsPerMessageType.

Initial values of maxThreadsPerMessageType are guesses and may need modifying based on real world results. Limiting threads may impact functionality, so this should be carefully tested.

Also be aware that the thread tracking may reduce network performance slightly, so be sure to test thoroughly on slower hardware.
2023-09-22 13:06:03 +01:00
AlphaX-Projects
cae3fdcb06 Update ElectrumX Servers 2023-09-22 11:12:50 +02:00
CalDescent
8b69b65712 Initial implementation of batch block reward distributions.
There are 3 values that need to be specified for this feature trigger:

- blockRewardBatchSize: the number of blocks per batch (1000 recommended).

- blockRewardBatchStartHeight: the block height at which the system switches to batch reward distributions. Must be a multiple of blockRewardBatchSize. Ideally this would be set to a block height that will occur at least 1-2 weeks after the release / auto update goes live.

- blockRewardBatchAccountsBlockCount: the number of blocks to include online accounts in the lead up to the batch reward distribution block (25 recommended).

Once active, rewards are no longer distributed every block and instead are distributed every 1000th block. This includes all fees from the distribution block and the previous 999 blocks.

The online accounts used for the distribution are taken from one of the previous 25 blocks (e.g. blocks xxxx975-xxxx999). The validation ensures that it is always the block with the highest number of online accounts in this range. If this number of online accounts is shared by multiple blocks, it will pick the one with the lowest height.

The idea behind 25 blocks is that it's low enough to reduce load in the other 975 blocks, but high enough for it to be extremely difficult for block signers to influence reward payouts.

Batch distribution blocks contain a copy of the online accounts from one of these 25 preceding blocks. However, online account signatures and online accounts timestamp are excluded to save space. The core will validate that the copy of online accounts is exactly matching those of the earlier validated block, and are therefore valid.

Fairly comprehensive unit tests have been written but this needs a lot more testing on testnets before it can be considered stable.

Future note: Once released, the online accounts mempow can optionally be scaled back so that it is no longer running 24/7, and instead is only running for 2-3 hours each day in the lead up to the batch distribution block. In each 1000 block cycle, the mempow would ideally start at least an hour before block xxxx975 is due to be minted, and continue for an hour after block xxxx000 (the reward distribution block). None of the ideas mentioned in this last paragraph are coded yet.
2023-09-17 15:04:37 +01:00
AlphaX-Projects
1fbb1659a3 Update dependencies 2023-09-13 10:43:05 +02:00
AlphaX-Projects
9959985a13 Update grpc 2023-09-09 10:56:17 +02:00
AlphaX-Projects
15c073edfe
Merge branch 'Qortal:master' into master 2023-09-08 15:20:44 +02:00
d453e80c6b
Update fee in qdn script 2023-09-08 03:30:06 -04:00
4fb799ba38
Merge pull request #125 from crowetic/master
ADD: (Tools) New features for qdn script
2023-09-08 03:26:34 -04:00
29e56158ae
Merge pull request #98 from Nuc1eoN/german-translation
ADD: (Language) German updates
2023-09-08 02:37:45 -04:00
f74c9672f6
Merge branch 'test-german' into german-translation 2023-09-08 02:29:07 -04:00
AlphaX-Projects
a7ca306d1b Update json dependency 2023-09-07 16:54:09 +02:00
AlphaX-Projects
1a9087984a Swagger updates 2023-09-06 17:22:13 +02:00
AlphaX-Projects
94e9f86245 Version 3.8.0 get faster the NTP offset 2023-09-06 09:11:41 +02:00
AlphaX-Projects
bd05578035 Reverse need latest 1.7.x 2023-09-05 18:31:32 +02:00
AlphaX-Projects
c0ed4022a5 Update log4j2.properties 2023-09-05 15:27:33 +02:00
AlphaX-Projects
12dbff79c9 Update logging properties 2023-09-05 15:20:21 +02:00
AlphaX-Projects
6be3897fdb Update logging 2023-09-05 15:06:46 +02:00
AlphaX-Projects
43921e6ab8 Update jetty server 2023-09-05 13:44:54 +02:00
AlphaX-Projects
b92c7cc866 Update dependencies and ntp servers 2023-09-05 11:35:53 +02:00
AlphaX-Projects
053d56d01d Update hsqldb to 2.7.2 , ciyam at to 1.4.1 2023-09-05 09:40:01 +02:00
CalDescent
eb6a834fd9 Default minPeerVersion set to 4.3.0 2023-09-01 10:47:45 +01:00
CalDescent
a08f10ece3 Bump version to 4.3.0 2023-08-25 16:13:23 +01:00
CalDescent
ad51073f25 mempowTransactionUpdatesTimestamp set to Fri Sep 01 2023 09:00:00 UTC 2023-08-25 16:11:43 +01:00
CalDescent
5983e6ccc9 Merge branch 'add-create-bytes-endpoint' 2023-08-25 16:05:35 +01:00
CalDescent
760788e82b Updated tools. 2023-08-25 12:12:32 +01:00
CalDescent
d39131ffa9
Merge pull request #128 from kennycud/master
consolidated shared functionality into ACCTTests.java
2023-08-25 11:17:52 +01:00
CalDescent
3fbcc50503 Fixed deserialization issues with CreationRequest, added validation, and made small code tweaks for consistency with other endpoints. 2023-08-25 11:01:48 +01:00
26ac7e5be5 consolidated shared functionality into ACCTTests.java 2023-08-24 15:47:45 -07:00
b051f9be89 fix strings in request 2023-08-23 14:41:50 -05:00
24ff3ab581 base64 to byte array 2023-08-22 22:46:58 -05:00
34382b6e69 add endpoint 2023-08-22 19:58:24 -05:00
CalDescent
086a9afa0e Testnet mempowTransactionUpdatesTimestamp set to 1692554400000 2023-08-20 16:52:49 +01:00
CalDescent
9428f9688f
Merge pull request #126 from jschulthess/master
Fix website sub-folder rendering 404
2023-08-20 14:22:58 +01:00
CalDescent
9f4a0b7957
Merge pull request #127 from kennycud/master
consolidated shared functionality into BitcoinyTests.java
2023-08-20 14:22:18 +01:00
CalDescent
3c8574a466 Trade bot improvements
- Add async responder thread from @catbref which was previously only in place for LTC
- Log when computing mempow nonces
- Skip transaction import if signature is invalid
- Added checks to a message test to mimic trade bot transaction lookups
2023-08-20 12:42:49 +01:00
Jürg Schulthess
c428d7ce2e
Merge branch 'Qortal:master' into master 2023-08-20 09:21:48 +02:00
CalDescent
4feb8f46c8 Updated testchain.json to use new unitFees structure. 2023-08-19 20:42:15 +01:00
CalDescent
1f7a60dfd8 Fixed long term issue preventing trade bot statuses from being logged correctly. 2023-08-19 20:27:05 +01:00
CalDescent
379b850bbd Increase default maxTransactionsPerBlock to 50 now that the process is faster. Can be increased much further in the future. 2023-08-19 14:12:12 +01:00
CalDescent
7bb61ec564 Update various transaction types at a future unknown timestamp.
- PUBLICIZE transactions are no longer possible.
- ARBITRARY transactions are now only possible using a fee.
- MESSAGE transactions only confirm when they are being sent to an AT. Messages to regular addresses (or no recipient) will expire after 24 hours.
- Difficulty for confirmed MESSAGE transactions increases from 14 to 16.
- Difficulty for unconfirmed MESSAGE transactions decreases from 14 to 12.
2023-08-19 13:57:26 +01:00
CalDescent
b0224651c2 Always use rate limiter for metadata requests, and sleep for a random amount of time between fetching metadata items. 2023-08-18 20:32:44 +01:00
CalDescent
6bf2b99913 Wait until unconfirmed transactions are considered to be valid before broadcasting them. 2023-08-18 15:37:24 +01:00
Jürg Schulthess
fd9d0c4e51
Merge branch 'Qortal:master' into master 2023-08-13 19:17:11 +02:00
kennycud
c2756a5872
Merge branch 'Qortal:master' into master 2023-08-13 05:32:47 -07:00
CalDescent
ecfb9a7d6d Bump version to 4.2.4 2023-08-12 19:34:59 +01:00
CalDescent
dd9d3fdb22 Add to the unconfirmed transactions cache when importing a transaction. 2023-08-12 19:27:19 +01:00
CalDescent
eea2884112 Bump version to 4.2.3 2023-08-12 18:59:00 +01:00
CalDescent
640a472876 Unit fee increase set to 1692118800000 2023-08-12 18:55:27 +01:00
CalDescent
e244a5f882 Removed legacy code that is no longer needed. 2023-08-12 16:09:41 +01:00
CalDescent
278dca75e8 Increase minimum fee at a future undecided timestamp. 2023-08-12 15:18:29 +01:00
CalDescent
897c44ffe8 Optimized transaction importing by using a temporary cache of unconfirmed transactions. 2023-08-12 11:14:21 +01:00
CalDescent
d9147b3af3 Cache the online accounts validation result, to speed up block minting. 2023-08-12 10:24:55 +01:00
fe840bbf02 consolidated shared functionality into BitcoinyTests.java 2023-08-08 12:17:29 -07:00
CalDescent
133848ef50 Speed up status rebuilding by excluding transactions that aren't hosted locally by the node. 2023-08-05 19:42:30 +01:00
CalDescent
e44b38819e Specify table name in query, to avoid potential ambiguity with indexes. 2023-08-05 13:17:30 +01:00
CalDescent
eecd37d6bc Speed up finding arbitrary transactions. 2023-08-05 13:10:03 +01:00
CalDescent
a3ab5238d3 Merge branch 'master' into arbitrary-resources-cache 2023-08-05 13:01:00 +01:00
Jürg Schulthess
e7901a391f
Merge branch 'Qortal:master' into master 2023-08-03 21:32:06 +02:00
CalDescent
9574100a08 Bump version to 4.2.2 2023-08-02 21:36:57 +01:00
CalDescent
528583fe38 Added logging relating to unconfirmed transactions. 2023-08-02 21:32:57 +01:00
CalDescent
33cfd02c49 Fixed issues in commit f5c8dfe 2023-08-02 21:13:33 +01:00
Jürg Schulthess
18e880158f
Merge branch 'Qortal:master' into master 2023-08-01 11:09:06 +02:00
CalDescent
94d3664cb0 Bump version to 4.2.1 2023-07-31 19:30:45 +01:00
CalDescent
f5c8dfe766 Added maxTransactionsPerBlock setting (default 25) to reduce minting load on slower machines.
This is a short term limit, is well above current usage levels, and can be increased substantially in future once the block minter code has been improved.
2023-07-31 19:25:26 +01:00
Jürg Schulthess
a3526d84bc
Merge branch 'Qortal:master' into master 2023-07-30 13:39:44 +02:00
CalDescent
f7e1f2fca8 Increased timeout for SEARCH_QDN_RESOURCES from 10 to 30 seconds. 2023-07-28 21:47:29 +01:00
CalDescent
811b647c88 Catch UnsupportedAddressTypeException and fall back to IPv4 binding. 2023-07-28 18:58:47 +01:00
CalDescent
3215bb638d More online accounts improvements 2023-07-22 10:44:41 +01:00
CalDescent
8ae7a1d65b Removed (Get)OnlineAccountsV1 and V2, as these are no longer used. 2023-07-21 14:28:47 +01:00
CalDescent
29dcd53002 Revert "Improved filtering of online accounts data."
This reverts commit c14fca5660c5dc0b833075ef7e978a919965b19d.
2023-07-16 20:04:45 +01:00
CalDescent
62908f867a Bump version to 4.2.0 2023-07-16 19:09:08 +01:00
Jürg Schulthess
ff7a87ab18
Merge branch 'Qortal:master' into master 2023-07-10 20:53:24 +02:00
CalDescent
5f86ecafd9 Refactored developer proxy, and modified IPv6 fallback so that it only occurs on a connection failure. 2023-07-09 12:35:46 +01:00
CalDescent
9694094bbf Sanitize inputs used for the working path when building arbitrary data, and throw/handle an exception if it still doesn't work.
Should fix issue on Windows systems due to reserved characters in certain resource names.
2023-07-08 14:27:24 +01:00
CalDescent
d8237abde5 Don't update statuses when processing arbitrary transactions, to improve success rate and speed it up. 2023-07-08 14:16:50 +01:00
CalDescent
fe999a11f4 Include CANCEL_SELL_NAME transactions when performing a complete rebuild of names. 2023-07-08 11:06:33 +01:00
CalDescent
c14fca5660 Improved filtering of online accounts data. 2023-07-08 11:05:14 +01:00
Jürg Schulthess
cc8cdcd93a Fix website sub-folder rendering 404 2023-07-03 09:52:07 +02:00
CalDescent
537779b152 Use a separate repository instance when updating caches. 2023-07-02 17:49:49 +01:00
crowetic
ac433b1527
Add files via upload 2023-06-27 13:29:10 -07:00
CalDescent
fd8d720946 Added support for group encryption in service validation. 2023-06-23 13:32:23 +01:00
CalDescent
c0eeef546a Added support for group encryption in service validation. 2023-06-23 13:30:10 +01:00
CalDescent
badd6ad2b0 Added optional minLevel filter to GET /arbitrary/resources/search and the SEARCH_QDN_RESOURCES action. 2023-06-23 11:55:49 +01:00
CalDescent
b4794ada72 Merge branch 'master' into arbitrary-resources-cache 2023-06-17 14:17:49 +01:00
CalDescent
d628b3ab2a Renamed the "usePrefix" parameter to "includeResourceIdInPrefix", and slightly modified its functionality. 2023-06-17 13:04:46 +01:00
CalDescent
5928b54a33 Added developer QDN proxy.
This allows Q-Apps and websites to be developed and tested in real time, by proxying an existing webserver such as localhost:5173 from React/Vite. The proxy adds all QDN functionality to the existing server in real time. Needs UI integration before all features can be used.
2023-06-17 13:03:29 +01:00
CalDescent
4b04b99401 Discard changes before setting status. 2023-06-16 14:59:07 +01:00
CalDescent
7e872f7800 Update QDN cache when receiving a metadata file as part of a resource download. 2023-06-16 14:58:33 +01:00
91dfc5efd0
Merge pull request #119 from QuickMythril/upgrade-tls
Upgraded to TLSv1.3
2023-06-14 10:25:25 -04:00
1343a88ee3
Merge pull request #124 from QuickMythril/translate-jp
Added Japanese translations (Credit: R M)
2023-06-02 08:01:04 -04:00
7f7b02f003 Updated Japanese translations (Credit: R M) 2023-06-02 06:17:48 -04:00
5650923805
Merge pull request #123 from QuickMythril/2023-05-25
Added some fee info & Q-App support for foreign coins
2023-05-31 09:14:46 -04:00
CalDescent
5fb2640a3a Bump version to 4.1.3 2023-05-30 18:05:05 +01:00
CalDescent
66c91fd365 MIN_PEER_VERSION for handshake set to 4.1.1 2023-05-30 17:42:31 +01:00
CalDescent
bfc03db6a9 Default minPeerVersion set to 4.1.2 2023-05-30 17:41:39 +01:00
a4bb445f3e Added Japanese translations for review 2023-05-29 04:23:22 -04:00
27afcf12bf Prepared files for Japanese translations 2023-05-29 04:07:52 -04:00
CalDescent
707176a202 Improved detection of an existing arbitrary resources cache. 2023-05-27 11:30:19 +02:00
CalDescent
74a914367f Merge branch 'master' into arbitrary-resources-cache
# Conflicts:
#	src/main/java/org/qortal/controller/Controller.java
#	src/main/java/org/qortal/repository/RepositoryManager.java
#	src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java
2023-05-26 19:37:13 +02:00
CalDescent
eda6ab5701 Fixed some failing unit tests, and ignored some failing BTC ones that have been superseded by LTC. 2023-05-26 18:01:09 +02:00
13da0e8a7a Adjusted fee info to long format 2023-05-25 17:26:29 -04:00
d260c0a9a9 Updated info on foreign coin fees 2023-05-25 16:35:08 -04:00
655073c524 Added 2m timeout for GET_WALLET_BALANCE action 2023-05-25 04:41:03 -04:00
crowetic
c8f3b6918f
Update start.sh
Added better defaults for JVM_MEMORY_ARGS and description as to how and when to uncomment the line.
2023-05-24 16:20:05 -07:00
CalDescent
1565a461ac Bump version to 4.1.2 2023-05-24 20:01:18 +01:00
CalDescent
1f30bef4f8 defaultArchiveVersion set to 2 2023-05-24 19:54:36 +01:00
CalDescent
6f0479c4fc Default minPeerVersion set to 4.1.1 2023-05-24 19:47:37 +01:00
CalDescent
b967800a3e Default repositoryConnectionPoolSize set to 240 2023-05-24 19:47:25 +01:00
CalDescent
0b50f965cc Default maxNetworkThreadPoolSize set to 120 2023-05-24 19:47:10 +01:00
CalDescent
90f7cee058 Bump version to 4.1.1 2023-05-21 20:34:04 +01:00
CalDescent
947b523e61 Limit query to 100 so that it doesn't return endless amounts of transaction signatures.
Using a separate database method for now to reduce risk of interfering with other parts of the code which use it. It can be combined later when there is more testing time.
2023-05-21 20:33:33 +01:00
CalDescent
95d72866e9 Use a better method to detect if a transactions table in need of a rebuild.
Should handle cases where a previous rebuild didn't fully complete, or missed a block.
2023-05-21 20:06:09 +01:00
CalDescent
aea1cc62c8 Fixed off-by-one bug (correctly this time) 2023-05-21 20:02:58 +01:00
CalDescent
c763445e6e Log to console if an extra core restart is needed to complete the update process (this needed ins some cases after bootstrapping). 2023-05-21 19:51:22 +01:00
CalDescent
7a6b83aa22 Bump version to 4.1.0 2023-05-21 16:49:59 +01:00
CalDescent
ba555174ba Merge branch 'master' into block-sequence 2023-05-21 15:35:50 +01:00
CalDescent
3763035d4a Default recoveryModeTimeout increased to 24 hours for now.
It doesn't quite work as intended, so it's best that it doesn't interfere right away. 24 hours should be long enough for any issues to be manually resolved.
2023-05-21 15:34:27 +01:00
CalDescent
b1a904a3c7 MIN_PEER_VERSION set to 4.0.0 2023-05-21 15:26:49 +01:00
CalDescent
3c4c5a1457 Default minPeerVersion set to 4.0.0 2023-05-21 15:22:24 +01:00
CalDescent
648fa66f6a Increased default maxPeers to 40. 2023-05-21 15:22:00 +01:00
CalDescent
072aa469e3 Reduce default minBlockchainPeers to 3, ahead of the upcoming reshape. 2023-05-21 15:21:04 +01:00
CalDescent
2b2d6f4e52 Updated message. 2023-05-21 14:02:45 +01:00
CalDescent
c6456669e2 Don't allow core to start if transaction sequences haven't been rebuilt yet. 2023-05-21 12:33:37 +01:00
CalDescent
a74fa15d60 Missing import 2023-05-21 12:31:49 +01:00
CalDescent
68b99c8643 Update status when rebuilding transaction sequences. 2023-05-21 12:28:51 +01:00
CalDescent
b9015217de Fixed bug causing final block to be missed in the reshape. 2023-05-21 11:05:34 +01:00
CalDescent
e1043ceacb Fixed bug causing duplicate AT entries in local array. 2023-05-21 08:41:56 +01:00
CalDescent
8b51590844 Include AT transactions when rebuilding transaction sequences, as these aren't directly included in the block archive. 2023-05-20 20:54:22 +01:00
CalDescent
a8d92805f9 Added extra check for topOnly mode. 2023-05-20 11:33:43 +01:00
CalDescent
2cc5b90306 Merge branch 'master' into block-sequence 2023-05-14 17:28:27 +01:00
CalDescent
f5f82dc3f6 Fixed issues relating to using a separate repository instance when determining the latest status of a resource. 2023-05-13 19:20:18 +01:00
CalDescent
633f73aa86 Removed API key fields from documentation for methods that don't require an API key by default. 2023-05-13 16:32:33 +01:00
CalDescent
a49529ad9b Cache updating moved back to existing threads when processing or importing a transaction, to remove chances of queued updates being lost.
The dedicated cache manager thread is now used for metadata updates only. If metadata ever goes missing from the db, it would be straightforward to have a background thread that corrects any discrepancies between the filesystem and the db. Not adding that until it is needed.
2023-05-13 15:11:32 +01:00
CalDescent
f451bccbf6 Fixed bug causing descriptions to be truncated in the cache. 2023-05-13 14:54:00 +01:00
CalDescent
4cb755a2f1 Added GET /stats/supply/circulating API endpoint, to fetch total QORT minted so far. 2023-05-13 13:48:27 +01:00
CalDescent
5ed3237d2f Clear queue before exiting cache manager thread. 2023-05-13 13:36:40 +01:00
CalDescent
5c7d12f25e Fixed bug causing incorrect creation dates in the cache. 2023-05-13 12:27:56 +01:00
CalDescent
92119b5558 Increased per-name limit for followed names by 4x. 2023-05-12 20:14:14 +01:00
CalDescent
23d211836f Fixed case sensitivity issue when updating status in the cache. 2023-05-12 20:10:51 +01:00
CalDescent
36a731255a Automatically delete cached resources & metadata if there is no longer a latest transaction. 2023-05-12 20:08:53 +01:00
CalDescent
b661d39844 Cache updating moved to a dedicated thread.
Hopeful fix for serialization failures which occurred when updating from various different network threads.
2023-05-12 19:39:31 +01:00
CalDescent
7725c5e21f Always ignore unsupported services when building the cache. 2023-05-12 12:03:32 +01:00
CalDescent
21f01226e9 Merge branch 'master' into arbitrary-resources-cache 2023-05-12 11:50:24 +01:00
CalDescent
8a1bf8b5ec Return full name data in GET /names. 2023-05-12 11:41:15 +01:00
CalDescent
f8233bd05b Added optional after parameter to GET /names. 2023-05-12 11:41:00 +01:00
CalDescent
29480e5664 Added SEARCH_NAMES Q-App action. 2023-05-12 11:17:09 +01:00
CalDescent
5a873f9465 Added prefix parameter to GET /names/search. 2023-05-12 11:11:34 +01:00
CalDescent
dc1289787d Ignore per-name limits when using storagePolicy ALL. 2023-05-12 10:12:38 +01:00
CalDescent
ba4866a2e6 Added GET /crosschain/tradeoffers/hidden endpoint, to show offers that are currently being hidden.
This uses the maxTradeOfferAttempts setting, so modifying this setting will affect the number of offers that are returned.
2023-05-12 10:01:38 +01:00
CalDescent
2cbc5aabd5 Added maxTradeOfferAttempts setting (default 3).
Offers with more than 3 failures will be hidden from the API and websocket, to prevent unbuyable offers from staying in the order books and continuously failing. maxTradeOfferAttempts can be optionally increased on a node to show more trades that would otherwise be hidden.
2023-05-12 09:59:30 +01:00
e3be43a1e6
Changed get name API call to use reduced name 2023-05-11 12:31:00 -04:00
1e10bcf3b0
Merge branch 'Qortal:master' into upgrade-tls 2023-05-09 15:38:20 -04:00
a575ea4423
Merge pull request #120 from QuickMythril/get-votes-api
Created get votes API call
2023-05-09 15:34:54 -04:00
3e45948646 Added get votes option to return only counts 2023-05-08 23:41:31 -04:00
49c0d45bc6 Added count to get votes API call 2023-05-08 23:26:23 -04:00
cda32a47f1 Added API call to get votes 2023-05-08 20:23:54 -04:00
CalDescent
49063e54ec Bump version to 4.0.3 2023-05-08 19:18:38 +01:00
CalDescent
df3c68679f Log the action to the console, instead of the entire event. 2023-05-08 14:43:00 +01:00
CalDescent
c210d63c40 Added "mode" parameter to GET /arbitrary/resources/search, with possible values of LATEST, ALL.
By default, only the latest resource is returned for a name/service combination. All identifiers can be optionally returned by setting `mode` to "ALL".

More search modes can be added in the future, for instance "RELEVANT" or "POPULAR" (these are just ideas, and are not currently supported).
2023-05-08 13:41:23 +01:00
CalDescent
0ec661431c Added optional "before" and "after" params to GET /arbitrary/resources/search 2023-05-08 12:46:15 +01:00
CalDescent
8fa344125c Fixed issue updating cache when receiving metadata via the network. 2023-05-08 12:34:26 +01:00
CalDescent
2fd5bfb11a Support title/description metadata searching in GET /arbitrary/resources/search
"query" searches name, identifier, title and description fields
"title" searches title only
"description" searches description only

All support "&prefix=true", to indicate searching by prefix only.
2023-05-08 12:34:26 +01:00
CalDescent
cdcb268bd9 Exclude status if includeStatus != true 2023-05-08 12:34:26 +01:00
CalDescent
d03a2d7da9 Resource statuses moved to the db, so they don't have to be calculated on demand for every API call. 2023-05-08 12:34:26 +01:00
CalDescent
961aa9eefd Show splash screen when building QDN cache. 2023-05-08 12:34:26 +01:00
CalDescent
865d3d8aff Fixed ordering, to keep consistency with existing approach. 2023-05-08 12:34:26 +01:00
CalDescent
c0f29f848f Fixed more bugs. 2023-05-08 12:34:26 +01:00
CalDescent
94f4c501fa Update caches where possible when processing arbitrary transactions. 2023-05-08 12:34:26 +01:00
CalDescent
200b0f3412 Added POST /arbitrary/resources/cache/rebuild endpoint to allow a rebuild of the cache. 2023-05-08 12:34:25 +01:00
CalDescent
eb7a29dd2e Fixed bugs. 2023-05-08 12:34:25 +01:00
CalDescent
9dba4b2968 Initial attempt at a database cache to hold arbitrary resources and metadata. 2023-05-08 12:34:25 +01:00
CalDescent
81788610c4 Merge branch 'master' of github.com:Qortal/qortal 2023-05-08 12:18:34 +01:00
CalDescent
fc10b61193 Fixed slow validation issue caused by loading the entire resource into memory. 2023-05-08 12:17:44 +01:00
CalDescent
05b4ecd4ed Updated documentation. 2023-05-08 12:16:17 +01:00
CalDescent
aba589c0e0 Added optional "build" parameter to GET_QDN_RESOURCE_STATUS.
This triggers an async build when checking the status.
2023-05-08 12:15:53 +01:00
CalDescent
c682fa89fd Avoid duplicate concurrent QDN builds. 2023-05-08 12:14:00 +01:00
CalDescent
21d1750779 Added more debug logging when building resources. 2023-05-08 12:13:12 +01:00
CalDescent
923e90ebed Fixed occasional NPE 2023-05-08 12:12:40 +01:00
catbref
9490c62242 Improved tx.pl that supports local signing via openssl and "deploy_at" transaction type + other minor fixes 2023-05-08 12:07:02 +01:00
CalDescent
c941bc6024 Catch and log all exceptions when publishing data. 2023-05-07 11:19:42 +01:00
CalDescent
0acf0729e9 Bump version to 4.0.2 2023-05-06 15:10:46 +01:00
CalDescent
1f77ee535f Added link to example Q-App projects. 2023-05-06 12:16:59 +01:00
CalDescent
b693a514fd Fixed warnings, and other improvements. 2023-05-06 12:13:41 +01:00
CalDescent
b571931127 Fixed formatting of services list 2023-05-05 22:35:19 +01:00
CalDescent
92b983a16e Q-Apps documentation updates. 2023-05-05 22:25:12 +01:00
CalDescent
3f71a63512 Increased timeout for other new actions. 2023-05-05 18:30:14 +01:00
CalDescent
86b5bae320 Set timeout of PUBLISH_MULTIPLE_QDN_RESOURCES to 60 mins. 2023-05-05 13:22:14 +01:00
CalDescent
3775135e0c Added helper methods to fetch lists of private or public service objects.
These can ultimately be used to help inform the cleanup manager on the best order to delete files when the node runs out of space. Public data should be given priority over private data (unless the node is part of a data market contract for that data - this isn't developed yet).
2023-05-05 12:39:11 +01:00
CalDescent
c172a5764b Added _PRIVATE services, to allow for publishing/validation of encrypted data.
New additions:

QCHAT_ATTACHMENT_PRIVATE
ATTACHMENT_PRIVATE
FILE_PRIVATE
IMAGE_PRIVATE
VIDEO_PRIVATE
AUDIO_PRIVATE
VOICE_PRIVATE
DOCUMENT_PRIVATE
MAIL_PRIVATE
MESSAGE_PRIVATE
2023-05-05 12:26:18 +01:00
CalDescent
1a5e3b4fb1 Added GET /names/search endpoint, to search names via case insensitive, partial name matching. 2023-05-05 11:24:52 +01:00
CalDescent
f39b6a15da Fixed refresh bug on Windows. 2023-05-05 11:03:13 +01:00
CalDescent
2dfee13d86 Remove all backslashes from vars in HTML parser (correct order this time) 2023-05-03 19:44:54 +01:00
CalDescent
b9d81645f8 Revert "Remove all backslashes from vars in HTML parser."
This reverts commit 9547a087b25401183c0ca7d87c3a757858bdd862.
2023-05-03 19:40:17 +01:00
CalDescent
9547a087b2 Remove all backslashes from vars in HTML parser. 2023-05-03 19:38:31 +01:00
CalDescent
e014a207ef Escape all vars added by HTML parser 2023-05-03 19:28:26 +01:00
CalDescent
611240650e Added GET /chat/messages/count endpoint, which is identical to /chat/messages but returns a count of the messages rather than the messages themselves. 2023-05-03 19:27:59 +01:00
CalDescent
c71dce92b5 Bump version to 4.0.1 2023-05-01 19:34:01 +01:00
CalDescent
34c3adf280 Limit MAIL and MESSAGE to 1MB. 2023-04-29 19:04:17 +01:00
CalDescent
95a1c6bf8b Added "encoding" parameter to the SEARCH_CHAT_MESSAGES action. 2023-04-29 17:48:58 +01:00
CalDescent
36e944d7e2 Added MAIL and MESSAGE services. 2023-04-29 17:45:38 +01:00
CalDescent
f044166b81 More qdnBase improvements, to hopefully handle all cases correctly. 2023-04-29 17:13:50 +01:00
CalDescent
aed1823afb Added support of simple Range headers when requesting QDN data. 2023-04-28 20:36:06 +01:00
CalDescent
6dfaaf0054 Set charset to UTF-8 in various places that bytes are converted to a string. 2023-04-28 13:06:29 +01:00
CalDescent
45bc2e46d6 Improved metadata trimming, to better handle multibyte UTF-8 characters. 2023-04-28 12:48:38 +01:00
CalDescent
46e2e1043d Fixed issue with <base href> introduced in v4.0.0 2023-04-28 12:18:27 +01:00
CalDescent
a3518d1f05 Revert "Fixed bug with base path."
This reverts commit ce52b3949501cccf66240093a077686b9f48c664.
2023-04-28 12:13:31 +01:00
CalDescent
0a1ab3d685 Added GET_QDN_RESOURCE_METADATA action. 2023-04-28 10:57:04 +01:00
CalDescent
5dbacc4db3 Added "Accept-Ranges" header when serving arbitrary data.
Allows for video seeking when using URL playback, even though the Range header isn't implemented yet. This could be heavily optimized by adding full support of the Range/Content-Range headers, however this is still a big step forward as it allows for (inefficient) seeking.
2023-04-28 10:12:16 +01:00
CalDescent
1ce2dcfb2b Fixed bug which prevented qortal:// URLs from working properly in most cases. 2023-04-25 08:33:33 +01:00
CalDescent
ed6333f82e Allow for faster and more frequent retries when QDN data fails to be retrieved (thanks to suggestions from @xspektrex) 2023-04-23 19:14:28 +01:00
CalDescent
f27c9193c7 Auto delete any metadata files that are unreadable (e.g. due to being empty, or invalid JSON). 2023-04-23 11:30:42 +01:00
CalDescent
e48529704c Bump version to 4.0.0 2023-04-22 16:08:09 +01:00
CalDescent
53508f9298 Fixed problems in last commit. 2023-04-22 11:33:59 +01:00
CalDescent
33aeec7e87 Added various new service types, in preparation for Q-Apps release. 2023-04-22 11:00:21 +01:00
8f847d3689 Upgraded to TLSv1.3 2023-04-21 19:30:29 -04:00
CalDescent
16dc23ddc7 Added new actions to gateway handler. 2023-04-21 21:45:16 +01:00
CalDescent
e80494b784 Fixed unit test. 2023-04-21 20:22:18 +01:00
CalDescent
111ec3b483 Fixed typo 2023-04-21 20:05:24 +01:00
CalDescent
db4a9ee880 Return "Resource does not exist" error if requesting a non-existent resource via GET_QDN_RESOURCE_URL. 2023-04-21 19:50:01 +01:00
CalDescent
b1ebe1864b Fixed bug in error handling. 2023-04-21 19:27:24 +01:00
CalDescent
3c251c35ea Fixed divide by zero error in GET /arbitrary/resource/status/* 2023-04-21 18:21:41 +01:00
CalDescent
4954a1744b Fixed case sensitivity bugs. 2023-04-21 17:47:29 +01:00
a7bbad17d7
Merge pull request #118 from QuickMythril/temp-test
Added missing parameter to test
2023-04-21 11:06:09 -04:00
8ca9423c52 Added missing parameter to test 2023-04-21 10:58:09 -04:00
CalDescent
32b9b7e578 Use a temporary file when reading on-chain data. 2023-04-21 13:59:29 +01:00
CalDescent
f045e10ada Removed all case sensitivity when searching names. 2023-04-21 12:56:15 +01:00
CalDescent
560282dc1d Added "exactMatchNames" parameter to GET /arbitrary/resources/search 2023-04-21 12:55:59 +01:00
CalDescent
9cd6372161 Improved GET /admin/settings/{setting} further, in order to support all settings (fixes ones such as bitcoinNet). 2023-04-21 12:06:16 +01:00
CalDescent
2370a67b8a Merge branch 'master' into q-apps 2023-04-21 11:07:01 +01:00
CalDescent
0993903aa0 Added GET /settings/{setting} endpoint
Based on work by @QuickMythril, but modified to be generic.
2023-04-21 11:03:24 +01:00
CalDescent
f5e9b91d6b
Merge pull request #116 from QuickMythril/restart
Added API call for restarting node
2023-04-21 10:27:58 +01:00
CalDescent
7fe507a497
Merge pull request #117 from QuickMythril/rm-swagger-check
Removed 3rd-party swagger server validation
2023-04-21 10:26:19 +01:00
CalDescent
10f12221c9 Fixed exception in readJson(), and removed some duplicated code. 2023-04-21 09:42:04 +01:00
85980e4cfc Removed 3rd-party swagger server validation 2023-04-20 16:41:47 -04:00
7bb6b84e86 Added API call for restarting node 2023-04-20 16:23:57 -04:00
CalDescent
dc25d33739 Merge branch 'master' into q-apps 2023-04-19 20:57:31 +01:00
CalDescent
358e67b050 Added "bindAddressFallback" setting, which defaults to "0.0.0.0".
Should fix problems on systems unable to use IPv6 wildcard (::) for listening, and avoids having to manually specify "bindAddress": "0.0.0.0" in settings.json.
2023-04-19 20:56:47 +01:00
CalDescent
8331241d75 Bump version to 3.9.1 2023-04-18 19:01:45 +01:00
CalDescent
e041748b48 Improved name rebuilding code, to handle some more complex scenarios. 2023-04-16 13:59:25 +01:00
CalDescent
06691af729
Merge pull request #113 from QuickMythril/drew-api
Added API calls to get node settings & get, create, and vote on polls
2023-04-16 11:43:31 +01:00
CalDescent
cfe6dfcd1c If nameFilter contains an empty or nonexistent list, return an empty array. 2023-04-15 18:27:55 +01:00
CalDescent
3f00cda847 "nameListFilter" added to LIST_QDN_RESOURCES and SEARCH_QDN_RESOURCES Q-Apps actions. 2023-04-15 16:02:25 +01:00
CalDescent
a286db2dfd "namefilter" param in GET /arbitrary/resources/search is now exact match, which makes more sense when filtering results by names in a list. 2023-04-15 15:55:52 +01:00
CalDescent
28bd4adcd2 Removed GET /arbitrary/resources/names endpoint, as it's unused and doesn't scale well. 2023-04-15 15:42:47 +01:00
CalDescent
61b7cdd025 Added "followedonly" and "excludeblocked" params to GET /arbitrary/resources and GET /arbitrary/resources/search, as well as LIST_QDN_RESOURCES and SEARCH_QDN_RESOURCES Q-Apps actions. 2023-04-15 15:24:10 +01:00
CalDescent
250245d5e1 Added new list management actions to Q-Apps documentation. 2023-04-15 14:34:30 +01:00
CalDescent
0258d2bcb6 Fixed layout issues recently introduced in documentation. 2023-04-15 14:31:41 +01:00
735de93848 Removed internal use parameter from API endpoint 2023-04-15 09:25:28 -04:00
57485bfe36 Removed check from poll tx that creator is owner 2023-04-15 09:11:27 -04:00
CalDescent
ed05560413 Gateway auth alert box replaced with a modal overlay in the lower right hand corner of the screen. 2023-04-15 10:11:33 +01:00
CalDescent
892b667f86 Fixed console errors seen in certain cases. 2023-04-15 09:57:26 +01:00
CalDescent
ea7a2224d3 Allow the name of a list to be specified as a "namefilter" param in GET /arbitrary/resources/search. Any names in the list will be included in the search (same as if they were specified manually via &name=). 2023-04-14 17:44:06 +01:00
CalDescent
20893879ca Allow multiple name parameters to optionally be included in GET /arbitrary/resources/search
Also updated SEARCH_QDN_RESOURCES action, to allow multiple names to be optionally specified via the "names" parameter.
2023-04-14 17:17:05 +01:00
CalDescent
b08e845dbb Updated docs to include sending of foreign coins 2023-04-14 16:24:27 +01:00
CalDescent
e60cd96514 Fixed occasional NPE seen in ArbitraryDataFileMessage 2023-04-14 11:02:27 +01:00
CalDescent
e2a2a1f956 Fixed bug with GET_QDN_RESOURCE_URL action. 2023-04-11 19:03:56 +01:00
CalDescent
7f53983d77 Added support for hash routing in URL shown in address bar. 2023-04-09 18:21:19 +01:00
CalDescent
ce52b39495 Fixed bug with base path. 2023-04-09 17:55:41 +01:00
CalDescent
3296779125 Update address bar when navigating within an app. 2023-04-09 17:11:20 +01:00
CalDescent
3dcd9d237c Added "_qdnBase" global javascript var, for apps to use as a basename / path prefix. 2023-04-07 19:03:28 +01:00
23ec71d7be Renamed API calls from "voting" to "polls" 2023-04-06 04:48:18 -04:00
5bbde4dcdb Added API calls to get polls & node settings 2023-04-06 04:41:51 -04:00
dc2da8b283
Merge pull request #37 from DrewMPeacock/VotingTransactions
Fix up CREATE_POLL and VOTE_ON_POLL transactions to process and valid…
2023-04-06 03:35:58 -04:00
f3772d19f5
Merge pull request #36 from DrewMPeacock/master
Add API handles to build CREATE_POLL and VOTE_ON_POLL transactions.
2023-04-06 03:34:50 -04:00
CalDescent
35def54ecc Added support for multiple block/follow lists.
Any list with the following prefix will be used in block/follow logic:

blockedNames
blockedAddresses
followedNames

For instance, any names in a list named "blockedNames_CustomBlockList" would also be blocked, along with those in the standard "blockedNames" list.

This will ultimately allow apps to offer custom block/follow lists to users (once list functionality is added to the Q-Apps API).
2023-04-02 14:42:49 +01:00
CalDescent
2086a2c476 Moved block/follow utility methods to a new ListUtils class 2023-04-02 10:58:16 +01:00
CalDescent
4835e5732d Fixed issue which caused UI to lock up when using qortalRequest() 2023-04-02 10:06:02 +01:00
CalDescent
d831972005 Fixed NPE in isMetadataEqual() 2023-03-31 18:41:58 +01:00
CalDescent
f6914821d3 Always use PUT for on-chain data. 2023-03-31 18:30:05 +01:00
CalDescent
073d124aef Add "percentLoaded" to resource statuses. 2023-03-31 17:27:04 +01:00
CalDescent
a83e332c11 Major upgrade of arbitrary data functionality, to support on-chain data for small payloads.
Max size for on-chain data is 239 bytes, due to 16-byte IV. Must be a single file resource, without .qortal folder.
2023-03-31 15:53:08 +01:00
CalDescent
7deb9328fa Don't delete metadata when deleting a resource from the Data Management screen. 2023-03-31 15:36:51 +01:00
CalDescent
e598d7476b Updated documentation, to discourage custom timeouts. 2023-03-31 14:58:30 +01:00
CalDescent
85735fabb2 Block external links. 2023-03-31 13:03:46 +01:00
CalDescent
7392082875 Ignore "qdnAuthBypassEnabled" setting when in gateway mode. 2023-03-31 09:48:25 +01:00
CalDescent
88f8041b05 Updated testnet documentation. 2023-03-31 09:47:38 +01:00
CalDescent
3109c3bb16 Increased timeouts to 5 mins for various actions that require the user to confirm. 2023-03-29 18:47:03 +01:00
CalDescent
8d462dedfa Added routing info to documentation. 2023-03-28 19:18:21 +01:00
CalDescent
fdd9741936 Documentation updates. 2023-03-26 21:19:10 +01:00
CalDescent
929d0ac897 Added "name" filter to GET /arbitrary/resources and LIST_QDN_RESOURCES. 2023-03-24 10:31:35 +00:00
CalDescent
952d18390b Q-Apps documentation updates 2023-03-24 09:53:49 +00:00
CalDescent
bc026d9d1c Merge branch 'master' into q-apps 2023-03-22 21:48:18 +00:00
CalDescent
ea2577d1c3 Include "created" and "updated" timestamps in GET /arbitrary/resource/* API endpoints. 2023-03-22 21:44:32 +00:00
CalDescent
c78593cf15 Reduced MAX_DESCRIPTION_LENGTH from 500 to 240, in preparation for upcoming arbitrary db reshape. 2023-03-22 21:40:26 +00:00
CalDescent
de4523c34e Added support for custom URL routing when using the APP service.
Unhandled requests (where no file exists) are now forwarded to the index file, to allow for custom routing in the app. This applies to the APP service only. For the WEBSITE and other services, unhandled requests will return a 404. In future, we may be able to allow websites to opt in to URL routing too, and maybe even allow both services to specify custom routing rules in a file.
2023-03-22 21:38:02 +00:00
CalDescent
b08329dcf1 Bump version to 3.9.0 2023-03-22 20:14:11 +00:00
CalDescent
668be633c4 arbitraryOptionalFeeTimestamp set to Friday, 31st March 2023 at 16:00:00 2023-03-22 19:58:51 +00:00
CalDescent
ea6225ab9a Include "mimeType" in metadata for single file resources (but only when a metadata file would have otherwise been created). 2023-03-19 16:27:57 +00:00
CalDescent
055b66e835 Fixed gateway prefix bugs. 2023-03-19 11:02:49 +00:00
CalDescent
2a7a2d3220 Added gateway-specific Q-Apps handler. For now, just show a warning alert if an app requires authentication / interactive features. 2023-03-19 10:41:37 +00:00
CalDescent
73a7c1fe7e More improvements to Service handling. 2023-03-19 10:18:13 +00:00
CalDescent
2848ae695c More improvements to Service handling. 2023-03-19 10:17:56 +00:00
CalDescent
713fd4f0c6 Added GET_QDN_RESOURCE_PROPERTIES Q-App action. 2023-03-19 08:56:06 +00:00
CalDescent
519bb10c60 Updated docs for PUBLISH_QDN_RESOURCE, to include "filename" parameter. 2023-03-18 18:15:28 +00:00
CalDescent
3a64336d9f If the MIME type can't be determined from the file's contents, fall back to using the filename. 2023-03-18 17:57:07 +00:00
CalDescent
5ecc633fd7 GET /arbitrary/resource/properties/{service}/{name}/{identifier} can now extract the MIME type from the file's contents as an alternative to using the filename. 2023-03-18 17:50:13 +00:00
CalDescent
1b9afce21f Filename API renamed to GET /resource/properties/{service}/{name}/{identifier}.
Now returns filename, size, and mimeType where available.
2023-03-18 16:39:23 +00:00
CalDescent
f9f34a61ac Treat service as an int in other parts of ArbitraryTransactionData too 2023-03-18 15:44:01 +00:00
CalDescent
46b225cdfb Treat service as an int in other parts of ArbitraryTransactionData too 2023-03-18 15:18:36 +00:00
CalDescent
4ce3b2a786 Added GET /resource/filename/{service}/{name}/{identifier} endpoint.
This allows the filename of single file resources to be returned via the API. Useful to help determine to file format of the data.
2023-03-18 15:16:41 +00:00
CalDescent
87ed49a2ee Added optional "filename" parameter when publishing data from a string or base64-encoded string.
This causes the data to be stored with the requested filename, instead of generating a random one. Also, randomly generated filenames now use a timestamp instead of a random number.
2023-03-18 15:11:53 +00:00
CalDescent
a555f503eb Treat service as an int in ArbitraryTransactionData 2023-03-18 10:41:53 +00:00
CalDescent
50780aba53 Set max size of APP service to 50MB. 2023-03-18 10:41:14 +00:00
CalDescent
2bee3cbb5c Treat service as an int in ArbitraryTransactionData 2023-03-18 10:40:27 +00:00
CalDescent
534a44d0ce Fixed bugs with URL building. 2023-03-17 22:58:14 +00:00
CalDescent
469c1af0ef Added new search features to the SEARCH_QDN_RESOURCES action.
Existing action renamed to LIST_QDN_RESOURCES, which is an alternative for listing QDN resources without using a search query.
2023-03-17 22:11:34 +00:00
CalDescent
5656100197 Added "identifier", "name", and "prefix" parameters to GET /arbitrary/resources/search endpoint.
- "identifier" is an alternative to "query" that will search identifiers only.
- "name" is an alternative to "query" that will search names only.
- "query" remains the same as before - it searches both name and identifier fields.
- "prefix" is a boolean, and when true it will only match the beginning of each field. Works with "identifier", "name", and "query" params.
2023-03-17 19:47:57 +00:00
CalDescent
d9cac6db39 Allow "data:" URLs to be played in app/website media players.
E.g: src="data:video/mp4;base64,VideoContentEncodedInBase64GoesHere"
2023-03-17 19:33:41 +00:00
CalDescent
98b0b1932d Merge branch 'master' into q-apps 2023-03-17 13:17:47 +00:00
CalDescent
9968865d0e Updated parsing of "encoding" in websockets, for consistency with other params. 2023-03-17 13:17:23 +00:00
CalDescent
05eb337367 Added optional limit/offset/reverse query string params to GET /websockets/chat/messages.
Without this, the websocket returns all messages on connection, which is very time consuming.
2023-03-17 13:15:57 +00:00
CalDescent
5386db8a3f Added ping/pong functionality to CHAT websockets. 2023-03-17 13:11:01 +00:00
CalDescent
edae7fd844 Added optional "encoding" query string param for various chat APIs and websockets, as base58 is too slow for the amount of data it is now processing.
Usage:
Add `encoding=BASE64` query string parameter to opt in to base64 encoding of returned chat data. Defaults to BASE58 for backwards support.

Compatible endpoints:
GET /chat/messages
GET /chat/message/{signature}
GET /chat/active/{address}
GET /websockets/chat/active/*
GET /websockets/chat/messages
2023-03-17 12:46:14 +00:00
CalDescent
4840804d32 Fixed qdn utility usage docs. 2023-03-17 10:22:26 +00:00
CalDescent
a4551245cb Improved error logging in BlockArchiveUtils.importFromArchive() 2023-03-12 19:08:57 +00:00
CalDescent
e4f45c1a70 Break out of orphan loop when stopping. 2023-03-12 19:08:07 +00:00
CalDescent
bc44b998dc The transaction sequences reshape now fetches transactions from the archive.
This is required as it's the only place that holds the original order of each block's transactions. We cannot sort them, because the comparator function for transactions has some dependencies on the existing order for AT transactions. As a result, topOnly nodes cannot perform this reshape, and will be unable to run this version.
2023-03-10 21:29:35 +00:00
CalDescent
b89a35ac69 Merge branch 'master' into block-sequence
# Conflicts:
#	src/main/java/org/qortal/controller/Controller.java
#	src/main/java/org/qortal/repository/RepositoryManager.java
2023-03-10 19:52:05 +00:00
CalDescent
b5cb5f1da3 Fixed bug causing cache invalidation to be skipped, due to incorrect message reuse.
The "Data Management" screen should now update correctly without a core restart.
2023-03-10 19:46:58 +00:00
CalDescent
101023ba1d Updated link. 2023-03-10 16:39:14 +00:00
CalDescent
ed73162881 Merge branch 'master' into q-apps 2023-03-10 15:41:31 +00:00
CalDescent
0388626e42 Use a lower file size target (10MB instead of 100MB) when using archive V2, as the average block size is over 90% smaller. 2023-03-10 15:41:07 +00:00
CalDescent
c5c0dcf0f2 Testnet arbitraryOptionalFeeTimestamp set to Sun Mar 12 2023 at 12:00:00 UTC 2023-03-10 14:59:33 +00:00
CalDescent
384f592f59 Added testnet files to testnet/ directory.
This will be maintained with future feature triggers etc.
2023-03-10 14:59:27 +00:00
CalDescent
1528e05e0b Testnet arbitraryOptionalFeeTimestamp set to Sun Mar 12 2023 at 12:00:00 UTC 2023-03-10 14:29:52 +00:00
CalDescent
82c66c0555 Added testnet files to testnet/ directory.
This will be maintained with future feature triggers etc.
2023-03-10 14:28:13 +00:00
CalDescent
b5ce8d5fb3 Merge branch 'master' into q-apps
# Conflicts:
#	src/main/java/org/qortal/api/resource/ArbitraryResource.java
2023-03-10 14:03:08 +00:00
CalDescent
b4a736c5d2 Added optional "sender" filter to GET /chat/messages 2023-03-10 13:53:46 +00:00
CalDescent
4afbca7ed2 Merge branch 'rebuild-archive' 2023-03-10 11:50:09 +00:00
CalDescent
44aa0a6026 Catch ArithmeticException in block minter, so that it retries instead of giving up completely. 2023-03-10 10:00:30 +00:00
CalDescent
b1452bddf3 Added BlockArchiveV2 tests, and updated the V1 tests now that we no longer support bulk archiving/pruning 2023-03-06 17:17:55 +00:00
CalDescent
96ac883515 Throw exception and break out of loop if archive rebuilding fails 2023-03-06 14:40:17 +00:00
CalDescent
b6803490b9 Archive version is now loaded from the version of block 2 in the existing archive, or "defaultArchiveVersion" in settings if not available (default: 1). 2023-03-06 14:13:58 +00:00
CalDescent
3739920ad3 Added support for an optional fee in arbitrary transactions, to give the option for data to be published instantly (i.e. no proof of work / mempow required when fee is sufficient).
Takes effect at a future undecided timestamp.
2023-03-06 13:17:48 +00:00
CalDescent
7f21ea7e00 Added new bootstrap host 2023-03-05 13:16:58 +00:00
CalDescent
83b0ce53e6 Fixed bug in JSON validation. 2023-03-05 13:16:08 +00:00
CalDescent
d6ab9eb066 Rework of service validation, to allow a service to be specified as a single file resource.
This removes some complexity and duplication from custom validation functions. Q-Chat QDN functionality will need a re-test.
2023-03-05 11:39:53 +00:00
CalDescent
ac60ef30a3 Added JSON service, with a maximum size of 25KB, and a requirement that the data must be valid JSON. 2023-03-05 10:51:26 +00:00
CalDescent
94f14a39e3 Ensure theme is transferred when visiting a linked resource. 2023-03-03 18:16:35 +00:00
CalDescent
4b7844dc06 Pass the UI's theme to Q-Apps themselves, so they have the option of adapting to the user's theme.
Variable name is _qdnTheme, and possible values are "dark" or "light"
2023-03-03 17:55:46 +00:00
CalDescent
c40d0cc67b Same fix again but for multi file resources too. 2023-03-03 17:47:14 +00:00
CalDescent
3318093a4f Fixed preview functionality for resources other than websites/apps. 2023-03-03 17:33:15 +00:00
CalDescent
cf0681d7df Only rebuild if transaction has a name. 2023-03-03 17:10:45 +00:00
CalDescent
7d7cea3278 Only rebuild if transaction has a name. 2023-03-03 17:10:14 +00:00
CalDescent
7d38fa909d Rebuild name in ArbitraryTransaction.preProcess() 2023-03-03 16:15:10 +00:00
CalDescent
0b05de22a0 Rebuild name in ArbitraryTransaction.preProcess() 2023-03-03 16:14:43 +00:00
CalDescent
308196250e Updated documentation. 2023-03-03 16:13:49 +00:00
CalDescent
b254ca7706 Added support for optional Base64 encoding in FETCH_QDN_RESOURCE. 2023-03-03 15:39:37 +00:00
CalDescent
9ea2d7ab09 Updated documentation to remove an action that isn't supported in Q-Apps v1. 2023-03-03 14:24:10 +00:00
CalDescent
d166f625d0 Rework of preview mode.
All /arbitrary endpoints responsible for publishing data now support an optional "preview" query string parameter. If true, these endpoints will return a URL path to open the preview, rather than returning transaction bytes.
2023-03-03 14:20:45 +00:00
CalDescent
8e2dd60ea0 Increased default timeout for GET_USER_ACCOUNT from 30 seconds to 1 hour, to give the user more time to grant permissions. 2023-03-03 13:20:17 +00:00
CalDescent
d51f9368ef Fixed bug in HTML parser 2023-03-03 12:39:44 +00:00
CalDescent
b17035c864 Escape QDN vars and prefix with underscores. 2023-03-03 11:57:07 +00:00
CalDescent
fa14568cb9 Fixed issue causing "totalChunkCount" to exclude the metadata file in some cases.
ArbitraryDataFile now has a fileCount() method which returns the total number of files associated with that piece of data - i.e. chunks, metadata, and the complete file in cases where it isn't chunked.
2023-03-03 10:42:43 +00:00
CalDescent
64cd21b0dd Merge branch 'master' into q-apps 2023-02-28 22:03:19 +00:00
CalDescent
abdc265fc6 Removed legacy bulk archiving/pruning code that is no longer needed. 2023-02-26 16:54:14 +00:00
CalDescent
1153519d78 Various fixes as a result of moving to archive version 2. 2023-02-26 16:53:43 +00:00
CalDescent
0af6fbe1eb Added POST /repository/archive/rebuild endpoint to allow local archive to be rebuilt.
When "archiveVersion" is set to 2 in settings, this should allow the archive size to reduce by over 90%. Some nodes might want to maintain an older/larger version, for the purposes of development/debugging, so this is currently opt-in.
2023-02-26 16:52:48 +00:00
CalDescent
d54006caf7 Added "archiveVersion" setting, which specifies the archive version to be used when building. Defaults to 1 for now, but will bump to version 2 at the time of a wider rollout. 2023-02-26 15:59:18 +00:00
CalDescent
e1771dbaea Merge branch 'master' into rebuild-archive 2023-02-26 14:29:37 +00:00
CalDescent
9566bda279 Merge branch 'master' into block-sequence 2023-02-26 12:55:35 +00:00
CalDescent
cc98abeffb Reduced log spam 2023-02-26 12:51:52 +00:00
CalDescent
a3702ac6b0 Revert "Merge pull request #111 from AlphaX-Projects/master"
This reverts commit 69902f7f5b490257998a8a74cf9d26eece4a5680, reversing
changes made to 466c727dee8b6a4e5077170bcc5e5de14c88bf43.
2023-02-26 12:45:38 +00:00
CalDescent
c1ffe557e1 Fixed wording in marshaller exceptions. 2023-02-24 13:42:59 +00:00
CalDescent
c310a7c5e8 Added "X-API-VERSION" header support in POST /transactions/process.
Default is version "1". If version "2" is specified, the API will return the full transaction JSON on success, rather than just "true".

Example usage:

curl -X POST "http://localhost:12391/transactions/process" -H  "X-API-VERSION: 2" -d "signedTransactionBytesHere"
2023-02-24 13:41:52 +00:00
CalDescent
c5a0b00cde Q-Apps documentation updates based on UI development progress. 2023-02-24 12:15:22 +00:00
QuickMythril
69902f7f5b
Merge pull request #111 from AlphaX-Projects/master
Update hsqldb and grpc
2023-02-24 05:02:32 -05:00
AlphaX-Projects
999e8b8aca
Update pom.xml 2023-02-24 09:12:57 +01:00
CalDescent
466c727dee Bump version to 3.8.9 2023-02-22 19:01:10 +00:00
CalDescent
ba9f3b335c Added unit test to reproduce the UPDATE_NAME issue and prove that the fix is working correctly. 2023-02-22 18:59:43 +00:00
CalDescent
148ca0af05 Fixed long term bug with UPDATE_NAME transactions, causing name data to be incorrectly deleted if newName == name. 2023-02-22 09:16:52 +00:00
CalDescent
c39b9c764b Bump version to 3.8.8 2023-02-20 18:12:40 +00:00
CalDescent
d30eb6141a Default minPeerVersion set to 3.8.7 2023-02-20 18:10:21 +00:00
CalDescent
52c806f9e6 Bump version to 3.8.7 2023-02-19 22:44:59 +00:00
CalDescent
b2d31a7e02 Rebuild the name's history before processing a CancelSellNameTransaction. 2023-02-19 22:26:22 +00:00
CalDescent
cfa0b1d8ea Bump version to 3.8.6 2023-02-19 18:02:22 +00:00
CalDescent
edacce1bac Improved logging when creating bootstraps, and catch/log all exceptions. 2023-02-19 17:43:13 +00:00
CalDescent
074cba2266 Added QCHAT_AUDIO and QCHAT_VOICE services (limited to 10MB each) 2023-02-19 17:33:17 +00:00
CalDescent
7f23ef64a2 Updated /arbitrary/metadata/* response when not found. 2023-02-17 17:37:30 +00:00
CalDescent
5b7e9666dc Send URL updates to the UI when pages are loaded. 2023-02-17 15:40:06 +00:00
CalDescent
20d4e88fab Fixed API endpoints relying on getTransactionsFromSignature(), which therefore won't have worked properly since core V2. 2023-02-12 13:21:54 +00:00
CalDescent
a8c27be18a Modified AT and transaction repository queries to use Transactions.block_sequence instead of BlockTransactions.sequence.
The former is available for all blocks, whereas the latter is only available for unpruned blocks.

Also removed joins with the Blocks table - as the Blocks table is also pruned - and instead retrieved the height from the Transactions table.
2023-02-12 13:21:41 +00:00
CalDescent
af6be759e7 Fixed long term issue where logs would report "Repository in use by another process?" when the database actually failed to start for some other reason. It will now log the correct reason. 2023-02-12 13:20:31 +00:00
CalDescent
896d814385 Add block_sequence to Transactions table, and populate all past transactions.
This data was being lost when pruning the BlockTransactions table.

Note: on first run this will reshape the db, which can take several minutes.
2023-02-12 13:20:23 +00:00
QuickMythril
f4a32d19dd
Merge pull request #110 from QuickMythril/update-electrumx
Add new ElectrumX servers
2023-02-12 00:10:56 -05:00
QuickMythril
eb6d84c04d Add new ElectrumX servers 2023-02-12 00:10:13 -05:00
CalDescent
26587067d8
Merge pull request #109 from AlphaX-Projects/master
Add more data to active chats websocket
2023-02-11 13:25:13 +00:00
AlphaX-Projects
227d93a31e
Merge branch 'Qortal:master' into master 2023-02-11 13:39:26 +01:00
CalDescent
76f17dda53 Merge branch 'master' into rebuild-archive 2023-02-10 17:48:05 +00:00
CalDescent
830bae3dc1 Merge branch 'at-states-fix'
# Conflicts:
#	src/main/java/org/qortal/controller/repository/AtStatesPruner.java
#	src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java
2023-02-10 17:45:22 +00:00
CalDescent
ec09312cc5 Updated AdvancedInstaller project for 3.8.5 2023-02-10 17:42:12 +00:00
AlphaX-Projects
11654ba9c6 Add Chat Data 2023-02-10 11:05:54 +01:00
AlphaX-Projects
ea356d1026 add signatures to websockt 2023-02-10 10:27:28 +01:00
CalDescent
e7a3e511bd Bump version to 3.8.5 2023-02-08 19:37:01 +00:00
CalDescent
6fca30ce75 Added GET /admin/summary/alltime endpoint, to view a summary of chain activity since genesis. 2023-02-07 19:56:54 +00:00
CalDescent
e903e59f7f
Merge pull request #107 from QuickMythril/unused-address
Add unused foreign address to API calls
2023-02-06 18:43:22 +00:00
CalDescent
bef170df7e Updated PirateChain lightwallet servers. 2023-02-06 18:42:37 +00:00
QuickMythril
386bfa4e20
Merge pull request #108 from AlphaX-Projects/master
Add electrum servers and fix java reflect error
2023-02-05 07:38:30 -05:00
AlphaX-Projects
6f867031e2 Add electrum servers and fix java reflect error 2023-02-05 12:53:49 +01:00
QuickMythril
8f589391a6
Updated depreciated actions
Node.js 12 actions are deprecated. Please update the following actions to use Node.js 16: actions/checkout@v2, actions/cache@v2, actions/setup-java@v2. For more information see: https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/
2023-02-04 21:57:31 -05:00
QuickMythril
30c9f63cb1 Add unused foreign address to API calls
POST ​/crosschain​/{COIN}/unusedaddress
2023-02-04 21:03:55 -05:00
QuickMythril
952b21d9bd
Merge pull request #105 from QuickMythril/update-electrumx
Updated ElectrumX servers
2023-02-04 18:40:39 -05:00
QuickMythril
1f410a503e Updated ElectrumX servers 2023-02-04 18:30:31 -05:00
CalDescent
ae5b713e58 Rework of AT state trimming and pruning, in order to more reliably track the "latest" AT states.
This should fix an edge case where AT states data was pruned/trimmed but it was then later required in consensus. The older state was deleted because it was replaced by a new "latest" state in a brand new block. But once the new "latest" state was orphaned from the block, the old "latest" state was then required again.

This works around the problem by excluding very recent blocks in the latest AT states data, so that it is unaffected by real-time sync activity.

The trade off is that we could end up retaining more AT states than needed, so a secondary cleanup process may need to run at some time in the future to remove these. But it should only be a minimal amount of data, and can be cleaned up with a single query. This would have been happening to a certain degree already.

# Conflicts:
#	src/main/java/org/qortal/controller/repository/AtStatesPruner.java
#	src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java
2023-02-03 12:39:27 +01:00
CalDescent
257ca2da05 Bumped default block archive serialization version to V2. 2023-02-03 12:36:57 +01:00
CalDescent
d27316eb64 Clear cache after rebuilding. 2023-02-02 18:11:56 +01:00
CalDescent
64d8353629 Added V2 support in the block archive, and added feature to rebuild a V1 block archive using V2 block serialization. Should drastically reduce the archive size once rebuilt. 2023-02-02 15:54:03 +01:00
CalDescent
f5e30eeaf5
Merge pull request #104 from QuickMythril/foreign-height
Add foreign chain height to API calls
2023-02-01 20:28:55 +00:00
QuickMythril
21f5d9a3d0 Add foreign chain height to API calls
GET ​/crosschain​/{COIN}/height
2023-01-31 17:23:25 -05:00
CalDescent
3077810ea8 Fixed bugs causing websites to report as "Not published" when listed in the UI. 2023-01-29 18:05:04 +00:00
CalDescent
4ba2f7ad6a Small documentation updates 2023-01-29 17:20:25 +00:00
CalDescent
8eba0f89fe Added to Q-Apps documentation 2023-01-29 17:09:28 +00:00
CalDescent
600f98ddab Fixed bug in extractComponents() 2023-01-29 13:38:08 +00:00
CalDescent
eb07e6613f Fixed small bug 2023-01-29 13:23:12 +00:00
CalDescent
6c445ff646 GET_ACCOUNT_ADDRESS and GET_ACCOUNT_PUBLIC_KEY replaced with a single action: GET_USER_ACCOUNT, as it doesn't make sense to request address and public key separately (they are essentially the same thing). 2023-01-29 13:23:01 +00:00
CalDescent
4d9cece9fa Timeouts are specified by action, rather than using 10 second for every request. This allows certain requests to wait for longer before timing out, such as ones that create transactions. 2023-01-29 13:07:26 +00:00
CalDescent
8beffd4dae Switched to document.querySelectorAll() as otherwise we were only intercepting the first image on the page. 2023-01-29 12:12:47 +00:00
CalDescent
566c6a3f4b Added support for img src updates from a Q-App.
Example:

document.getElementById("logo").src = "qortal://thumbnail/QortalDemo/qortal_avatar";
2023-01-29 12:04:39 +00:00
CalDescent
1be3ae267e Reduce log spam. 2023-01-29 11:45:09 +00:00
CalDescent
7af551fbc5 Added "GET_QDN_RESOURCE_URL" Q-Apps action, to allow a website/app to programmatically determine the URL to retrieve any QDN resource it needs to access.
Examples:

### Get URL to load a QDN resource
```
let url = await qortalRequest({
    action: "GET_QDN_RESOURCE_URL",
    service: "THUMBNAIL",
    name: "QortalDemo",
    identifier: "qortal_avatar"
    // path: "filename.jpg" // optional - not needed if resource contains only one file
});
```

### Get URL to load a QDN website
```
let url = await qortalRequest({
    action: "GET_QDN_RESOURCE_URL",
    service: "WEBSITE",
    name: "QortalDemo",
});
```

### Get URL to load a specific file from a QDN website
```
let url = await qortalRequest({
    action: "GET_QDN_RESOURCE_URL",
    service: "WEBSITE",
    name: "AlphaX",
    path: "/assets/img/logo.png"
});
```
2023-01-29 11:44:59 +00:00
CalDescent
6ba6c58843 Added support for qortal:// protocol links when loading images from the DOM.
Example: <img src="qortal://THUMBNAIL/QortalDemo/qortal_avatar" />
2023-01-29 11:18:00 +00:00
CalDescent
ab34fae810
Merge pull request #90 from QuickMythril/german
Updated German translations
2023-01-28 20:22:11 +00:00
CalDescent
42f2d015b7
Merge branch 'master' into german 2023-01-28 20:22:02 +00:00
CalDescent
2181ece28d
Merge pull request #89 from lexandr0s/patch-2
Update ApiError_ru.properties
2023-01-28 20:21:13 +00:00
CalDescent
03a5d0e5f9
Merge pull request #88 from lexandr0s/patch-1
Update SysTray_ru.properties
2023-01-28 20:21:00 +00:00
CalDescent
352f094272
Merge pull request #99 from Nuc1eoN/polish-translation
Add polish translation
2023-01-28 20:20:35 +00:00
CalDescent
ca09dd264f Merge branch 'master' into q-apps 2023-01-28 20:14:35 +00:00
CalDescent
eea98d0bc7 Fixed bugs. 2023-01-28 18:37:04 +00:00
CalDescent
9c58faa7c2 Added LINK_TO_QDN_RESOURCE support in the gateway. 2023-01-28 18:36:55 +00:00
CalDescent
3cdfa4e276 Increased loading screen refresh interval from 1s to 2s. 2023-01-28 18:03:00 +00:00
CalDescent
380ba5b8c2 Show "File not found" on the loading screen when navigating to a non-existent resource. 2023-01-28 18:01:52 +00:00
CalDescent
04f248bcdd Upgraded gateway to support service and identifier.
The URL used to access the gateway is now interpreted, and the most appropriate resource is served. This means it can be used in different ways to retrieve any type of content from QDN. For example:

/QortalDemo
/QortalDemo/minting-leveling/index.html
/WEBSITE/QortalDemo
/WEBSITE/QortalDemo/minting-leveling/index.html
/APP/QortalDemo
/THUMBNAIL/QortalDemo/qortal_avatar
/QCHAT_IMAGE/birtydasterd/qchat_BfBeCz
/ARBITRARY_DATA/PirateChainWallet/LiteWalletJNI/coinparams.json
2023-01-28 17:56:24 +00:00
CalDescent
37b20aac66 Upgraded rendering to support identifiers, as well as single file resources.
This allows any QDN resource (e.g. an IMAGE) to be linked to from a website/app and then rendered on screen. It isn't yet supported in gateway or domain map mode, as these need some more thought.
2023-01-28 16:55:04 +00:00
CalDescent
e1e52b3165 RenderResource moved to restricted resources, as /render/* endpoints shouldn't ever need to be served over the gateway. 2023-01-28 15:52:46 +00:00
CalDescent
c5c826453b Removed unnecessary join when finding MESSAGE transactions, which caused secret to be unavailable when querying pruned blocks. 2023-01-28 15:41:48 +00:00
CalDescent
e86b9b1caf Added additional Litecoin ElectrumX server. 2023-01-28 15:34:30 +00:00
CalDescent
46e8baac98 Added linking between QDN websites / apps.
The simplest way to link to another QDN website is to include a link with the format:
<a href="qortal://WEBSITE/QortalDemo">link text</a>

This can be expanded to link to a specific path, e.g:
<a href="qortal://WEBSITE/QortalDemo/minting-leveling/index.html">link text</a>

Or it can be initiated programatically, via qortalRequest():

let res = await qortalRequest({
    action: "LINK_TO_QDN_RESOURCE",
    service: "WEBSITE",
    name: "QortalDemo",
    path: "/minting-leveling/index.html" // Optional
});

Note that qortal:// links don't yet support identifiers, so the above format is not confirmed.
2023-01-28 15:22:03 +00:00
CalDescent
3b6e1ea27f Added "qdnContext" variable, with possible values of "render", "gateway", or "domainMap".
This is used internally to allow Q-Apps to determine how to handle certain requests.
2023-01-28 14:42:29 +00:00
CalDescent
5a1cc7a0de Fixed/improved logging when an exception is caught whilst adding statuses to resources. 2023-01-28 14:32:17 +00:00
CalDescent
0ec5e39517 Fixed additional NPE 2023-01-28 14:31:04 +00:00
CalDescent
bede5a71f8 Fixed various NPEs when checking statuses of non-existent resources. 2023-01-28 14:17:23 +00:00
CalDescent
5e750b4283 Added new ArbitraryResourceStatus "NOT_PUBLISHED" - for when a non-existent resource is attempted to be loaded. 2023-01-28 14:15:54 +00:00
CalDescent
4a42dc2d00 Don't require prior authorization of QDN resources if qdnAuthBypassEnabled is true. Necessary for resource linking. 2023-01-28 14:14:44 +00:00
CalDescent
7fc170575c Merge branch 'cancel-sell-name-fixes' 2023-01-28 12:11:42 +00:00
CalDescent
876658256f Prevent a P2SH address being funded for a trade if there is an unconfirmed buy or cancel request in progress for it already.
This prevents foreign coins from leaving the local wallet when there is a high probability that the trade will fail, and therefore should reduce the chances of losing transaction fees due to refunds.

Whenever this occurs, the UI will show "Trade has an existing buy request or is pending cancellation." after clicking Buy.
2023-01-28 11:57:15 +00:00
CalDescent
a24ba40d5c Added additional Dogecoin ElectrumX server. 2023-01-28 09:54:15 +00:00
CalDescent
06d8a21714 Added CANCEL_SELL_NAME equivalents to NamesDatabaseIntegrityCheck.java 2023-01-27 19:38:26 +00:00
CalDescent
ae44065d7e Fixed issue with CancelSellName transactions. 2023-01-27 19:34:23 +00:00
CalDescent
6ad0989ea2 Reduce log spam 2023-01-27 18:35:44 +00:00
CalDescent
5962ebd08a More logging improvements in ArbitraryDataReader.decrypt() 2023-01-27 16:56:53 +00:00
CalDescent
bf06d47842 Create an ArbitraryDataResource object when building. Eventually this could be passed in to the reader instead of the individual components (service, name, identifier, etc)
This is now used to improve logging when extracting.
2023-01-27 16:55:43 +00:00
CalDescent
d7b1615d4f qdnAuthBypassEnabled defaulted to true, as it is needed for Q-Apps. 2023-01-27 16:26:36 +00:00
CalDescent
8c708558cb Implemented ElectrumX version negotiation. Fixes issues with DOGE wallet. 2023-01-27 14:33:34 +00:00
CalDescent
6b36d94c6f Removed searchResultsTransactions cache, to simplify code. The hostedTransactions cache is still in place, which limits disk reads when searching, so this additional cache isn't really needed. 2023-01-27 12:48:42 +00:00
CalDescent
8c41a4a6b3 Moved BootstrapResource to restricted resources 2023-01-22 21:08:42 +00:00
CalDescent
8dffe1e3ac Another rewrite of Q-App APIs, which removes the /apps/* redirects and instead calls the main APIs directly.
- All APIs are now served over the gateway and domain map, with the exception of /admin/*
- AdminResource moved to a "restricted" folder, so that it isn't served over the gateway/domainMap ports.
- This opens the door to websites/apps calling core APIs directly for certain read-only functions, as an alternative to using qortalRequest().
2023-01-22 18:59:46 +00:00
CalDescent
932a553b91 Merge branch 'master' into q-apps
# Conflicts:
#	src/main/java/org/qortal/api/resource/ArbitraryResource.java
2023-01-22 16:37:02 +00:00
CalDescent
1d568fa462 Return file lists via /arbitrary/metadata/* endpoints, but exclude it from /arbitrary/resources/* endpoints. 2023-01-22 16:29:23 +00:00
CalDescent
328ba48224 Merge branch 'master' into qdn-file-list 2023-01-22 16:12:54 +00:00
CalDescent
6196841609 Allow files without extensions in QCHAT_ATTACHMENT validation. 2023-01-22 15:59:16 +00:00
CalDescent
9f30571b12 Use a filename without an extension when publishing data from a string (instead of .tmp) 2023-01-22 15:58:53 +00:00
CalDescent
1f7fec6251 Exclude .qortal directory in validation functions, as it was incorrectly failing with "DIRECTORIES_NOT_ALLOWED". 2023-01-20 10:40:20 +00:00
CalDescent
c3f19ea0c1 Don't allow the custom validation methods to evade superclass validation. 2023-01-20 10:21:05 +00:00
CalDescent
e31515b4a2 Fixed bugs preventing single file GIF repositories and QCHAT attachments from passing validation. 2023-01-20 10:14:42 +00:00
CalDescent
8ad46b6344 Fixed/removed incorrect comments 2023-01-20 09:58:28 +00:00
CalDescent
57eacbdd59 Added "GET_PRICE" action. 2023-01-19 20:47:06 +00:00
CalDescent
86d6037af3 Added "SEARCH_TRANSACTIONS" action. 2023-01-19 20:22:29 +00:00
CalDescent
ca80fd5f9c Added "FETCH_BLOCK" and "FETCH_BLOCK_RANGE" Q-Apps actions. 2023-01-19 20:05:46 +00:00
CalDescent
03a54691a1 Merge branch 'master' into q-apps 2023-01-19 19:57:01 +00:00
CalDescent
3c8088e463 Removed all code duplication for Q-Apps API endpoints.
Requests are now internally routed to the existing API handlers. This should allow new Q-Apps API endpoints to be added much more quickly, as well as removing the need to maintain their code separately from the regular API endpoints.
2023-01-19 19:56:50 +00:00
CalDescent
2f7912abce Compute balances for Bitcoin-like coins using unspent outputs. Should fix occasional incorrect balance issue, and speed up loading time. 2023-01-18 19:30:43 +00:00
CalDescent
64529e8abf Added "reverse" and "includeOnlineSignatures" params to GET /blocks/range/{height} endpoint. 2023-01-18 19:04:54 +00:00
CalDescent
9d81ea7744 Bump version to 3.8.4 2023-01-16 20:26:00 +00:00
CalDescent
688acd466c Set checkpoint to block 1136300 2023-01-16 20:23:43 +00:00
CalDescent
81cf46f5dd Disable block signing on topOnly nodes. Minting rewards are still earned on topOnly for now. 2023-01-16 20:18:23 +00:00
CalDescent
4c52d6f0fc Fixed bug causing initial latestATStates data to be discarded. 2023-01-15 21:58:17 +00:00
CalDescent
de47a94677 Fixed bug causing initial latestATStates data to be discarded. 2023-01-15 15:51:10 +00:00
CalDescent
bd4c47dba6 Rework of AT state trimming and pruning, in order to more reliably track the "latest" AT states.
This should fix an edge case where AT states data was pruned/trimmed but it was then later required in consensus. The older state was deleted because it was replaced by a new "latest" state in a brand new block. But once the new "latest" state was orphaned from the block, the old "latest" state was then required again.

This works around the problem by excluding very recent blocks in the latest AT states data, so that it is unaffected by real-time sync activity.

The trade off is that we could end up retaining more AT states than needed, so a secondary cleanup process may need to run at some time in the future to remove these. But it should only be a minimal amount of data, and can be cleaned up with a single query. This would have been happening to a certain degree already.
2023-01-15 14:32:33 +00:00
CalDescent
c03f271825 Keep track of peers which are too divergent, and return an isTooDivergent boolean in /peers APIs.
isTooDivergent will be true or false if a definitive decision has been made, or missing from the response if not yet known. Therefore it should be safe to treat `"isTooDivergent": false` as a peer that is on the same chain.
2023-01-15 12:44:19 +00:00
CalDescent
dfe3754afc Block connections with peers older than 3.8.2, as those versions are nonfunctional due to recent feature triggers. 2023-01-15 12:07:27 +00:00
CalDescent
30105199a2 Default pruneBlockLimit increased from 1450 to 6000 (approx 5 days), to be more similar to the AT states retention time of full nodes. 2023-01-15 12:00:32 +00:00
CalDescent
e91e612b55 Added checkpoint lookup on startup.
Currently enabled for topOnly nodes only. This will detect if the node is on a divergent chain, and will force a bootstrap or resync (depending on settings) in order to rejoin the main chain.
2023-01-15 11:33:16 +00:00
CalDescent
2a55eba1f7 Updated AdvancedInstaller project for v3.8.3 2023-01-15 11:28:37 +00:00
CalDescent
39e59cbcf8 Bump version to 3.8.3 2023-01-14 18:47:46 +00:00
CalDescent
016191bdb0 Reduce log spam when a QDN resource can't be found due to it not being published. 2023-01-14 15:15:48 +00:00
CalDescent
0596a07c7d Reduced ArbitraryDataFileRequestThread count from 10 to 5, to reduce network flooding. 2023-01-14 12:58:35 +00:00
CalDescent
c62c59b445 Use correct timeout (12s) when sending arbitrary data to a peer, and improved logging. 2023-01-14 12:57:44 +00:00
CalDescent
f78101e9cc Updated a default bootstrap host to use a domain instead of its IP. 2023-01-14 11:07:54 +00:00
CalDescent
476fdcb31d Added serialization tests for chatReference, and grouped with other serialization tests into a single package. 2023-01-14 10:38:50 +00:00
CalDescent
02d5043ef7 Added missing calls to electrumX.setBlockchain(instance); for DGB and RVN. Thanks to @QuickMythril for noticing this. 2023-01-13 20:17:27 +00:00
CalDescent
0ad9e2f65b Added QCHAT_ATTACHMENT service, with custom validation function. 2023-01-13 20:08:47 +00:00
CalDescent
4dc0033a5a Added missing chatReferenceTimestamp in unit tests. 2023-01-13 19:45:52 +00:00
CalDescent
745cfe8ea1 chatReferenceTimestamp set to 1674316800000 (Sat, 21 Jan 2023 16:00:00 GMT) 2023-01-13 19:45:38 +00:00
CalDescent
6284a4691c Import test transactions as part of the serialization tests, to catch any issues with db schema data lengths. 2023-01-13 19:28:44 +00:00
CalDescent
41f88be55e Test serialization of CHAT transactions 2023-01-13 19:27:38 +00:00
CalDescent
ba95f8376f Increase CHAT transaction data limits to the maximum (4000 bytes) to allow for upcoming UI features. 2023-01-13 19:27:02 +00:00
CalDescent
8e97c05b56 Added missing feature trigger from unit tests. 2023-01-13 19:25:06 +00:00
CalDescent
2c78f4b45b Fixed typo and reworded "methods" to "actions", for consistency with the code. 2023-01-13 18:25:30 +00:00
CalDescent
613ce84df8 More documentation updates 2023-01-13 18:11:44 +00:00
CalDescent
2822d860d8 Fixed sample app 2023-01-13 18:01:38 +00:00
CalDescent
5a052a4f67 Documentation updates 2023-01-13 17:57:01 +00:00
CalDescent
32c2f68cb1 Initial APIs and core support for Q-Apps 2023-01-13 17:36:27 +00:00
CalDescent
4232616a5f Fixed QDN website preview functionality. 2023-01-13 12:07:24 +00:00
CalDescent
8ddcae249c Added gatewayLoopbackEnabled setting (default false) to allow serving gateway requests via localhost.
Useful for testing, but not recommended for production environments.
2023-01-13 12:05:57 +00:00
CalDescent
eb569304ba Improved refund/refundAll HTLC code, to handle cases where there have been multiple purchase attempts for the same AT. 2023-01-06 10:38:25 +00:00
CalDescent
b0486f44bb Added chat_reference index to speed up searches. 2023-01-02 17:47:36 +00:00
CalDescent
cecf28ab7b Merge branch 'chat-reference'
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
#	src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java
#	src/main/resources/blockchain.json
#	src/test/resources/test-chain-v2-block-timestamps.json
#	src/test/resources/test-chain-v2-disable-reference.json
#	src/test/resources/test-chain-v2-founder-rewards.json
#	src/test/resources/test-chain-v2-leftover-reward.json
#	src/test/resources/test-chain-v2-minting.json
#	src/test/resources/test-chain-v2-qora-holder-extremes.json
#	src/test/resources/test-chain-v2-qora-holder-reduction.json
#	src/test/resources/test-chain-v2-qora-holder.json
#	src/test/resources/test-chain-v2-reward-levels.json
#	src/test/resources/test-chain-v2-reward-scaling.json
#	src/test/resources/test-chain-v2-reward-shares.json
#	src/test/resources/test-chain-v2.json
2023-01-02 17:32:38 +00:00
CalDescent
98b92a5bf1 Introduced "historic threshold" to ARBITRARY transactions in order to save on verification times of older transactions.
This is based on the approach used for PUBLICIZE transactions.
2023-01-02 16:58:50 +00:00
CalDescent
6b45901c47 Fixed validation of existing reward share transactions. 2022-12-31 14:43:37 +00:00
CalDescent
166f9bd079 Bump version to 3.8.2 2022-12-24 21:28:02 +00:00
CalDescent
2f8f896077 Merge remote-tracking branch 'catbref/bugfix-deploy-at' 2022-12-24 16:01:23 +00:00
CalDescent
9a77aff0a6 Reduced difficulty of PUBLICIZE transactions from 15 to 14 (it is now the same as ARBITRARY transactions) 2022-12-24 14:10:49 +00:00
CalDescent
c6d65a88dc Increase mempow difficulty and threshold in ChatTransaction, to match the values in the UI. 2022-12-22 18:19:27 +00:00
CalDescent
4aea29a91b Improved PublicizeTransaction validation. 2022-12-22 18:03:29 +00:00
CalDescent
0e81665a36 Revert "Filter out peers of divergent or significantly inferior chains when syncing."
This reverts commit 1dc7f056f9cb9b08fb44ae50896844cfcd144ead. To be un-reverted in future when there is more time available for testing.
2022-12-22 15:10:19 +00:00
CalDescent
2a4ac1ed24 Limit to 250 CHAT messages per hour per account. 2022-12-22 15:09:04 +00:00
CalDescent
bb74b2d4f6 MAX_AVG_RESPONSE_TIME for ElectrumX servers increased from 0.5s to 1s. 2022-12-22 14:25:10 +00:00
CalDescent
758a02d71a Log Pirate light client server address if the wallet unable to be initialized. 2022-12-22 14:23:30 +00:00
CalDescent
7ae142fa64 Improved transaction validation. 2022-12-22 14:20:42 +00:00
CalDescent
a75ed0e634 Bump additional expandedAccount level references held in memory. 2022-12-22 14:18:39 +00:00
CalDescent
e40dc4af59 Fixed group ban expiry. 2022-12-22 14:16:57 +00:00
CalDescent
e678ea22e0 Fixed NPE in unit tests. Still need to work out how/when this was introduced. 2022-12-18 18:33:51 +00:00
CalDescent
cf3195cb83 Set "minAccountsToActivateShareBin" to 0 for certain tests. 2022-12-18 18:32:06 +00:00
CalDescent
80048208d1 Moved some test sponsorship utility methods to AccountUtils, so they can be used in other test classes too. 2022-12-15 12:14:42 +00:00
CalDescent
08de1fb4ec Disallow CHAT transactions with timestamps more than 5 minutes in the future. 2022-12-14 16:40:57 +00:00
CalDescent
99d5bf9103 Disallow transactions with timestamps more than 30 mins in the future (reduced from 24 hours) 2022-12-14 16:40:11 +00:00
CalDescent
1dc7f056f9 Filter out peers of divergent or significantly inferior chains when syncing. 2022-12-14 16:39:43 +00:00
CalDescent
cdeb2052b0 Bump version to 3.8.1 2022-12-08 18:26:34 +00:00
CalDescent
5c9109aca9 minPeerVersion set to 3.8.0 2022-12-08 18:25:19 +00:00
CalDescent
ccc1976d00 Added defensiveness 2022-12-08 18:25:03 +00:00
CalDescent
12fb6cd0ad onlineAccountMinterLevelValidationHeight moved forward to block 1092000 2022-12-08 18:24:34 +00:00
CalDescent
6f95e7c1c8 Bump version to 3.8.0 2022-12-05 21:57:32 +00:00
CalDescent
a69618133e Level 0 online account removals moved inside feature trigger, so it is coordinated with the new validation. 2022-12-05 21:34:26 +00:00
CalDescent
51ad0a5b48 onlineAccountMinterLevelValidationHeight set to 1093400 (approx 20 hours later) 2022-12-05 19:38:44 +00:00
CalDescent
45a6f495d2 selfSponsorshipAlgoV1Height set to 1092400 (approx 4pm UTC on Sat 10th December) 2022-12-05 19:38:26 +00:00
CalDescent
4d9964c080 Block connections with peers older than 3.7.0, as this has been released for long enough now. 2022-12-05 18:52:33 +00:00
CalDescent
9afc31a20d selfSponsorshipAlgoV1SnapshotTimestamp set to 1670230000000 2022-12-05 08:52:09 +00:00
CalDescent
d435e4047b SelfSponsorshipAlgoV1 2022-12-05 08:21:45 +00:00
CalDescent
c108afa27c Self sponsorship algo tests 2022-12-04 20:57:36 +00:00
CalDescent
eea42b56ee Added SelfSponsorshipAlgoV1Block, and call it when processing/orphaning a block at an undecided future height. 2022-12-04 18:21:01 +00:00
CalDescent
f4d20e42f3 Disallow TRANSFER_PRIVS transactions if the sending account has a penalty. Again, there will be no penalties until the algo runs, so it's safe without a feature trigger. 2022-12-04 11:54:05 +00:00
CalDescent
f14cc374c6 Include blocksMintedPenalty in effectiveBlocksMinted. This will be zero until the algo runs, so doesn't need a feature trigger. 2022-12-04 11:52:51 +00:00
CalDescent
99ba4caf75 We definitely can't retroactively validate minter levels, because there are confirmed cases in the chains history where this fails. Set to 999999999 until we have decided on a future block height. 2022-12-04 11:50:58 +00:00
catbref
ae991dda4d Fix creatorPublicKey not being unmarshaled when calling POST /at to deploy an AT 2022-11-28 21:52:37 +00:00
CalDescent
2b6ae57a27 Merge branch 'master' into chat-reference
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
#	src/main/resources/blockchain.json
#	src/test/resources/test-chain-v2-block-timestamps.json
#	src/test/resources/test-chain-v2-disable-reference.json
#	src/test/resources/test-chain-v2-founder-rewards.json
#	src/test/resources/test-chain-v2-leftover-reward.json
#	src/test/resources/test-chain-v2-minting.json
#	src/test/resources/test-chain-v2-qora-holder-extremes.json
#	src/test/resources/test-chain-v2-qora-holder-reduction.json
#	src/test/resources/test-chain-v2-qora-holder.json
#	src/test/resources/test-chain-v2-reward-levels.json
#	src/test/resources/test-chain-v2-reward-scaling.json
#	src/test/resources/test-chain-v2-reward-shares.json
#	src/test/resources/test-chain-v2.json
2022-11-27 20:06:55 +00:00
CalDescent
5ff7b3df6d hasInvalidSigner() now only checks the chain tip block, to reduce the amount of unintended side effects that can occur. 2022-11-27 19:59:46 +00:00
CalDescent
76686eca21 Modified reward share creation in test/common/AccountUtils to allow a fee to be specified. 2022-11-27 12:23:17 +00:00
CalDescent
3965f24ab5 Fixed bug 2022-11-26 19:06:02 +00:00
CalDescent
a75fd14e45 Added Account method needed for unit tests. 2022-11-26 17:23:43 +00:00
CalDescent
41cdf665ed Code tidy 2022-11-26 16:53:38 +00:00
CalDescent
6ea3c0e6f7 Give founder accounts as an effective minting level of 0 if they have a penalty. 2022-11-26 16:19:43 +00:00
CalDescent
5f0263c078 Modifications to block minting for unit tests, in order to solve an NPE and give more options to callers.
This shouldn't affect the behaviour of existing tests, other than an NPE being replaced with an assertNotNull().
2022-11-26 16:14:45 +00:00
CalDescent
58e5d325ff Added algo feature triggers to BlockChain.java (at future undecided block height & timestamp) 2022-11-26 16:10:18 +00:00
CalDescent
7003a8274b Added some API endpoints relating to penalties. Relies on some code not yet committed. 2022-11-26 15:51:06 +00:00
CalDescent
ab687af4bb Added new db query to fetch a list of all accounts that have created a non-self-share, based on confirmed transactions.
This will be used as the input dataset for the self sponsorship algo.
2022-11-26 12:21:43 +00:00
CalDescent
f50c0c87dd Account repository modifications for blocksMintedPenalty. 2022-11-26 12:03:13 +00:00
CalDescent
9c3a4d6e37 BlockChain.java additions for onlineAccountMinterLevelValidationHeight, which were missing from commit 68a0923 2022-11-26 11:55:27 +00:00
CalDescent
1c8a6ce204 When synchronizing, filter out peers that have a recent block with an invalid signer.
This avoids the wasted time and consensus confusion causes by syncing and then validation failing. This is significant after the algo has run, as many signers will become invalid.
2022-11-26 11:52:27 +00:00
CalDescent
68a0923582 Disallow level 0 minters in blocks, and exclude them when minting a new block.
The validation is currently set to a feature trigger of height 0, although this will likely be set to a future block, in case there are any cases in the chain's history where this validation may fail (e.g. transfer privs?)
2022-11-26 11:37:02 +00:00
CalDescent
617c801cbd Made Block.ExpandedAccount public, and added some more getters. This is needed for upcoming additional validation and unit tests. 2022-11-26 11:25:44 +00:00
CalDescent
b0c9ce7482 Add blocks minted penalty to Accounts table 2022-11-26 11:19:23 +00:00
CalDescent
4e829a2d05 Bump version to 3.7.0 2022-11-07 21:12:34 +00:00
CalDescent
a7402adfa5 Merge branch 'null-owned-groups' 2022-11-06 21:20:19 +00:00
CalDescent
9255df46cf Script updates to support add/remove dev group admins 2022-11-06 19:46:12 +00:00
CalDescent
db22445948 Include API key automatically in publish-auto-update-v5.pl 2022-11-06 14:52:14 +00:00
CalDescent
818e037e75 Merge branch 'master' into null-owned-groups 2022-11-06 13:08:54 +00:00
CalDescent
9c68f1038a Bump AT version to 1.4.0 2022-11-05 14:02:04 +00:00
CalDescent
10ae383bb6
Merge pull request #102 from catbref/faster-qort-buy-PoC
Proof of concept: speed up QORT buying
2022-11-01 18:55:21 +00:00
catbref
aead9cfcbf Proof of concept: speed up QORT buying
When users buy QORT ("Alice"-side), most of the API time is spent computing mempow for the MESSAGE sent to Bob's AT.
This is the final stage startResponse() and after Alice's P2SH is already broadcast.

To speed this up, the MESSAGE part is moved into its own thread allowing startResponse() to return sooner, improving the user experience.

Caveats:
If MESSAGE importAsUnconfirmed() somehow fails the the buy won't complete and Alice will have to wait for P2SH refund.
If Alice shuts down her node while MESSAGE mempow is being computed then it's possible the shutdown will be blocked until mempow is complete.

Currently only implemented in LitecoinACCTv3TradeBot as this is only proof-of-concept.
Tested with multiple buys in the same block.
2022-11-01 08:55:57 +00:00
CalDescent
055775b13d Include a list of files in the QDN metadata. 2022-10-30 18:54:38 +00:00
CalDescent
985c195e9e Added GIF_REPOSITORY, with custom validation function and unit tests. 2022-10-30 17:33:21 +00:00
CalDescent
0628847d14 Removed QORTAL_METADATA service tests. 2022-10-30 17:25:11 +00:00
CalDescent
4043ae1928 Added QCHAT_IMAGE service (with 500KB file size limit). 2022-10-30 17:23:46 +00:00
CalDescent
fa80c83864 Remove QORTAL_METADATA service as this uses its own protocol instead. 2022-10-30 17:07:56 +00:00
CalDescent
f739d8f5c6 Added increaseOnlineAccountsDifficultyTimestamp feature trigger to unit tests. 2022-10-28 18:06:34 +01:00
CalDescent
166425bee9 Added feature trigger timestamp (TBC) to increase online accounts mempow difficulty (also TBC). 2022-10-28 17:20:39 +01:00
CalDescent
59a804c560 Include "blocks remaining" in systray when syncing from more than 60 minutes away from a peer's chain tip. 2022-10-28 16:57:52 +01:00
CalDescent
b64c053531 Reuse the work buffer when verifying online accounts from the OnlineAccountsManager import queue.
This is a hopeful fix for extra memory usage since mempow activated, due to adding a lot of load to the garbage collector. It only applies to accounts verified from the import queue; the optimization hasn't been applied to block processing. But verifying online accounts when processing blocks is rare and generally would only last a short amount of time.
2022-10-28 16:54:53 +01:00
CalDescent
30cd56165a Speed up syncing blocks in the range of 1-12 hours ago by caching the valid online accounts. 2022-10-28 16:02:25 +01:00
CalDescent
510328db47 Removed unused timestamp value. 2022-10-28 15:50:43 +01:00
CalDescent
9d74f0eec0 Added haschatreference, with possible values of true, false, or null, to allow optional filtering by the presence or absense of a chat reference. 2022-10-24 19:21:29 +01:00
CalDescent
09014d07e0 Fixed issues retrieving chatReference from the db. 2022-10-23 19:29:31 +01:00
CalDescent
f83d4bac7b Reduced online accounts mempow difficulty to 5 on testnets.
This allows testnets to more easily coexist on the same machines that are running a mainnet instance, and still tests the mempow computation and verification in a non-resource-intensive way.
2022-10-23 17:01:58 +01:00
CalDescent
b3273ff01a Removed all mempow feature trigger conditionals.
We no longer need all the code complexity, now that 24 hours have passed since activation. We don't validate online accounts beyond 12 hours, and the data is trimmed after 24 hours.
2022-10-23 16:47:42 +01:00
CalDescent
1dd039fb2d Merge branch 'master' into chat-reference 2022-10-23 14:14:23 +01:00
CalDescent
1d5497e484 Modifications to support a single node testnet:
- Added "singleNodeTestnet" setting, allowing for fast and consecutive block minting, and no requirement for a minimum number of peers.
- Added "recoveryModeTimeout" setting (previously hardcoded in Synchronizer).
- Updated testnets documentation to include new settings and a quick start guide.
- Added "generic" minting account that can be used in testnets (not functional on mainnet), to simplify the process for new devs.
2022-10-23 14:13:38 +01:00
CalDescent
b37aa749c6 Removed onlineAccountsMemPoWEnabled setting as it's no longer needed. 2022-10-22 19:34:24 +01:00
CalDescent
e45ad37eb5 Fixed bug which could prevent invalid accounts being removed from the queue until the next valid one is added. 2022-10-22 19:30:08 +01:00
CalDescent
72985b1fc6 Reduce log spam, especially around the time of node startup before online accounts have been retrieved.
We expect a "Couldn't build a to-be-minted block" log on every startup due to trying to mint before having any accounts. This one has moved from error to info level because error logs can be quite intrusive when using an IDE.
2022-10-22 19:24:54 +01:00
CalDescent
6f27d3798c Improved online accounts processing, to avoid creating keys in the map before validation. 2022-10-22 19:18:41 +01:00
CalDescent
23a5c5f9b4 Fixed bug in original commit - we need to save the chat reference to the db. 2022-10-22 12:50:28 +01:00
CalDescent
a4759a0ef4 Re-ordered chat transaction transformation, to simplify UI code. New additions are now at the end of the data bytes. 2022-10-22 12:43:40 +01:00
CalDescent
910191b074 Added optional chatReference field to CHAT transactions.
This allows one message to reference another, e.g. for replies, edits, and reactions. We can't use the existing reference field as this is used for encryption and generally points to the user's lastReference at the time of signing.

"chatReference" is based on the "nameReference" field used in various name transactions, for similar purposes.

This needs a feature trigger timestamp to activate, and that same timestamp will need to be used in the UI since that is responsible for building the chat transactions.
2022-10-21 15:58:23 +01:00
CalDescent
57125a91cf Bump version to 3.6.4 2022-10-15 18:59:42 +01:00
CalDescent
3c565638c1 onlineAccountsMemoryPoWTimestamp set to Sat Oct 22 2022 16:00:00 UTC 2022-10-15 18:58:13 +01:00
CalDescent
c2d02aead9 Default minPeerVersion set to 3.6.3 2022-10-14 18:44:25 +01:00
CalDescent
0d9aafaf4e Reduced log spam 2022-10-14 17:03:10 +01:00
CalDescent
3844358380 Mark a peer as misbehaved if it fails to respond with a usable block 3 times in a row.
This should help to workaround deserialization and missing response issues.
2022-10-14 16:38:05 +01:00
CalDescent
b4125d2bf1 Fix for NPE in verifyMemoryPoW() 2022-10-14 11:34:46 +01:00
CalDescent
5c223179ed Updated AdvancedInstaller project for v3.6.3 2022-10-13 23:37:21 +01:00
CalDescent
f3cb57417a Merge branch 'master' of github.com:Qortal/qortal 2022-10-13 23:36:27 +01:00
CalDescent
7c7f071eba Bump version to 3.6.3 2022-10-12 08:54:27 +01:00
CalDescent
7c15d88cbc Fix for issue in BLOCK_SUMMARIES_V2 when sending an empty array of summaries.
The BLOCK_SUMMARIES message type would differentiate between an empty response and a missing/invalid response. However, in V2, a response with empty summaries would throw a BufferUnderflowException and be treated by the caller as a null message.

This caused problems when trying to find a common block with peers that have diverged by more than 8 blocks. With V1 the caller would know to search back further (e.g. 16 blocks) but in V2 it was treated as "no response" and so the caller would give up instead of increasing the look-back threshold.

This fix will identify BLOCK_SUMMARIES_V2 messages with no content, and return an empty array of block summaries instead of a null message.

Should be enough to recover any stuck nodes, as long as they haven't diverged more than 240 blocks from the main chain.
2022-10-12 08:52:58 +01:00
CalDescent
d4aaba2293 Bump version to 3.6.2 2022-10-10 19:06:08 +01:00
CalDescent
10d3176e70 Revert "Always use BlockSummariesMessage V1 (instead of V2) when responding to GetBlockSummaries requests."
This reverts commit 2d58118d7cfa717a4a6521b9d2fa2bd325c7e5ea.
2022-10-10 10:28:44 +01:00
CalDescent
36fcd6792a Discard BLOCK_SUMMARIES_V2 messages with an ID (thanks to @catbref for the code)
This is a better fix for the "contaminated chain tip summaries" issue. Need to reduce the logging level to debug before release.
2022-10-10 10:28:36 +01:00
CalDescent
cb1eee8ff5 GenericUnknownMessage.MINIMUM_PEER_VERSION set to 3.6.1.
This should ideally have been set in the 3.6.1 release, but not setting it is unlikely to have caused any problems.
2022-10-09 20:37:39 +01:00
CalDescent
2d58118d7c Always use BlockSummariesMessage V1 (instead of V2) when responding to GetBlockSummaries requests.
This should hopefully fix a potential issue where peer's chain tip data becomes contaminated with other summary data, causing incorrect sync decisions.
2022-10-09 20:11:01 +01:00
CalDescent
e6bb0b81cf Revert "Reduce INITIAL_BLOCK_STEP from 8 to 7."
This reverts commit 0088ba8485a73c723a9ea4555e0435d42df20a3f.
2022-10-09 19:11:20 +01:00
Nuc1eoN
8ddf4c9f9f Add polish translation 2022-10-09 15:35:19 +02:00
Nuc1eoN
0693e26cda Update/add German translation 2022-10-09 15:34:20 +02:00
CalDescent
77d60fc33f Revert "Skip GET_BLOCK_SUMMARIES requests if it can already be fulfilled entirely from the peer's chain tip block summaries cache."
This reverts commit 8cedf618f45a5d1ab24633e9b32972e663c3dcdd.
2022-10-09 14:11:28 +01:00
CalDescent
504f38b42a
Merge pull request #97 from Nuc1eoN/patch-1
Mark start/stop scripts as executables
2022-10-08 19:49:10 +01:00
Nuc1eoN
3a18599d85
Mark start/stop scripts as executables
The `start.sh` & `stop.sh` scripts have already been marked as executables in the source folder... But since we have only piped their contents, we need to set correct file permissions again.
2022-10-07 23:35:35 +02:00
CalDescent
0088ba8485 Reduce INITIAL_BLOCK_STEP from 8 to 7.
This allows the first pass to always be served from the peer's cache of 8 summaries. This allows a maximum of 7 to be returned, because the 8th spot is needed for the parent block's signature.
2022-10-07 14:47:46 +01:00
CalDescent
8cedf618f4 Skip GET_BLOCK_SUMMARIES requests if it can already be fulfilled entirely from the peer's chain tip block summaries cache.
Loading from the cache should speed up sync decisions, particularly when choose which peer to sync from. The greater the number of connected peers, the more significant this optimization will be. It should also reduce wasted network requests and data usage.

Adding this check prior to making a network request is a simple way to introduce the new cached summaries from BLOCK_SUMMARIES_V2 without having to rewrite a lot of the complex sync / peer comparison logic. Longer term we may want to rewrite that logic to read from the cache directly, but it doesn't make sense to introduce that level of risk at this point time, especially as the Synchronizer may be rewritten soon to prefer longer chains.

Even so, this is still quite a high risk commit so lots of testing will be needed.
2022-10-07 14:46:09 +01:00
CalDescent
fdd95eac56 Limit to 240 blocks in syncToPeerChain().
Should fix OutOfMemoryException often seen when syncing from 1000+ blocks behind the chain tip.
2022-10-07 11:05:24 +01:00
CalDescent
10b0f0a054 Catch JSON exceptions in PirateChainWalletController.
This could prevent additional wallets from being initialized if connection was lost while syncing an existing one.
2022-10-05 15:29:29 +01:00
CalDescent
1233ba6703 Bump version to 3.6.1 2022-10-04 20:08:30 +01:00
CalDescent
c35c7180d4 Return empty levels in GET /addresses/online/levels 2022-10-03 10:58:47 +01:00
CalDescent
7080b55aac Reintroduced initial sleep period in block archiver. 2022-09-25 19:43:56 +01:00
CalDescent
3890fa8490 Renamed constant for consistency 2022-09-25 18:46:33 +01:00
CalDescent
a9721bab3d Fixed issue causing startup of various components to be delayed by 30 seconds. 2022-09-25 18:39:56 +01:00
CalDescent
1bb8f1b6d2 Fixed bug in last commit.
We need to track items to remove separately from items to add, otherwise invalid accounts remain in the queue.
2022-09-25 12:36:00 +01:00
CalDescent
765416db71 Yet another attempt to optimize the online accounts import queue processing.
The main difference here is that we now remove items from the onlineAccountsImportQueue in a batch, _after_ they have been imported. This prevents duplicates from being added to the queue in the previous time gap between them being removed and imported.
2022-09-25 12:26:00 +01:00
CalDescent
5989473c8a Revert "Allow duplicate variations of each OnlineAccountData in the import queue, but don't allow two entries that match exactly."
This reverts commit 6d9e6e8d4c89582ffad23c1094b6a8e3aee91116.
2022-09-25 12:06:14 +01:00
CalDescent
aa9da45c01 Added optional filtering by reference in GET /chat/messages 2022-09-25 11:38:17 +01:00
CalDescent
4681218416 Include total count in debug trade presence logging 2022-09-24 15:49:29 +01:00
CalDescent
5c746f0bd9 Fixed bug which required a node to hold local trade presences before it would request any.
This caused large gaps with no presence data. They are removed when they expire, causing the local count to drop to zero, and the node would only start requesting them again once a peer had pushed one or more entries proactively.
2022-09-24 15:48:45 +01:00
CalDescent
309f27a6b8 Moved error to debug, as we now get a burst of these soon after startup, due to commit 99858f3.
This also shows that commit 99858f3 now prevents a block candidate with a very small number of online accounts being built immediately after startup.
2022-09-24 15:21:01 +01:00
CalDescent
d2ebb215e6 Fixed Synchronizer.getBlockSummaries() which was expecting BLOCK_SUMMARIES, but updated peers send BLOCK_SUMMARIES_V2 2022-09-24 14:36:49 +01:00
CalDescent
7a60f713ea Fixed error in rebase. 2022-09-24 14:35:02 +01:00
CalDescent
e80dd31fb4 BlockSummariesV2Message.MINIMUM_PEER_VERSION set to 3.6.1 2022-09-24 13:53:27 +01:00
catbref
94cdc10151 Initial work on BLOCK_SUMMARIES_V2, part of a bigger arc to improve synchronization.
Touches quite a few files because:

* Deprecate HEIGHT_V2 because it doesn't contain enough info to be fully useful during sync.
Newer peers will re-use BLOCK_SUMMARIES_V2.

* For newer peers, instead of sending / broadcasting HEIGHT_V2,
send top N block summaries instead, to avoid requests for minor reorgs.

* When responding to GET_BLOCK, and we don't actually have the requested block,
we currently send an empty BLOCK_SUMMARIES message instead of not responding,
which would cause a slow timeout in Synchronizer.

This pattern has spread to other network message response code,
so now we introduce a generic 'unknown' message type for all these cases.

* Remove PeerChainTipData class entirely and re-use BlockSummaryData instead.

* Each Peer instance used to hold PeerChainTipData - essentially single latest block summary - but now holds a List of latest block summaries.

* PeerChainTipData getter/setter methods modified for compatibility at this point in time.

* Repository methods that return BlockSummaryData (or lists of) now try to fully populate them,
including newly added block reference field.

* Re-worked Peer.canUseCommonBlockData() to be more readable

* Cherry-picked patch to Message.fromByteBuffer() to pass an empty, read-only ByteBuffer to subclass fromByteBuffer() methods, instead of null.
This allows natural use of BufferUnderflowException if a subclass tries to use read(), or hasRemaining(), etc. from an empty data-payload message.
Previously this could have caused an NPE.
2022-09-24 13:48:01 +01:00
CalDescent
863a5eff97 Moved various online accounts logs to TRACE level, to make it easier to monitor the queue processing when in DEBUG. 2022-09-24 13:11:28 +01:00
CalDescent
5b81b30974 Modified online accounts request interval, and introduced bursting.
It will now request online accounts every 1 minute instead of every 5 seconds, except for the first 5 minutes following a new online accounts timestamp, in which it will request every 5 seconds (referred to as the "burst" interval). It will also use the burst interval for the first 5 minutes after the node starts.

This is based on the idea that most online accounts arrive soon after a new timestamp begins, and so there is no need to request accounts so frequently after that. This should reduce data usage by a significant amount.

Once mempow is fully rolled out, the "burst" feature can be reduced or removed, since online accounts will be sent ahead of time, generally 15-30 mins prior to the new online accounts timestamp becoming active.
2022-09-24 13:02:27 +01:00
CalDescent
174a779e4c Add accounts from the import queue individually, and then skip future duplicates before unnecessarily validating them again.
This closes a gap where accounts would be moved from onlineAccountsImportQueue to onlineAccountsToAdd, but not yet imported. During this time, there was nothing to stop them from being added to the import queue again, causing duplicate validations.
2022-09-24 10:56:52 +01:00
CalDescent
c7cf33ef78 Set hasOurOnlineAccounts to true if one of our accounts is found before signing. 2022-09-24 10:23:55 +01:00
CalDescent
ea4f4d949b When validating online accounts, enforce mempow if the online account's timestamp is after the feature trigger. 2022-09-23 19:45:59 +01:00
CalDescent
6d9e6e8d4c Allow duplicate variations of each OnlineAccountData in the import queue, but don't allow two entries that match exactly. 2022-09-23 18:46:01 +01:00
CalDescent
99858f3781 Wait 30 seconds after the node starts before computing our online accounts.
This allows some time for initial online account lists to be retrieved, and reduces the chances of the same nonce being computed twice.
2022-09-23 18:28:41 +01:00
CalDescent
84a16157d1 Don't add online accounts to the import queue if they are already validated 2022-09-23 18:02:46 +01:00
CalDescent
49d83650f4 Removed online accounts V2 and V1 messaging, as the V3 format will soon be required due to the nonce values. 2022-09-23 15:25:44 +01:00
CalDescent
951c85faf1 Fixed bug causing error 500 in some cases. 2022-09-20 22:26:30 +01:00
CalDescent
84d42b93e1 Reordered code in Block.mint() to fix potential issue after mempow activates. 2022-09-20 08:50:37 +01:00
CalDescent
b99b1f5d57 Bump version to 3.6.0 2022-09-19 17:29:26 +01:00
CalDescent
952c51ab25 QORA / block reward adjustments set to activate at height 1010000 2022-09-19 17:27:07 +01:00
CalDescent
64ef8ab863 OnlineAccountsV3Message.MIN_PEER_VERSION set to 3.6.0 2022-09-19 16:36:39 +01:00
CalDescent
93fd80e289 Require that add/remove admin transactions can only be created by group members.
For regular groups, we require that the owner adds/removes the admins, so group membership is adequately checked. However for null-owned groups this check is skipped. So we need an additional condition to prevent non-group members from issuing a transaction for approval by the group admins.
2022-09-19 16:34:31 +01:00
CalDescent
5581b83c57 Added initial admin approval features for groups owned by the null account.
* The dev group (ID 1) is owned by the null account with public key 11111111111111111111111111111111
 * To regain access to otherwise blocked owner-based rules, it has different validation logic
 * which applies to groups with this same null owner.
 *
 * The main difference is that approval is required for certain transaction types relating to
 * null-owned groups. This allows existing admins to approve updates to the group (using group's
 * approval threshold) instead of these actions being performed by the owner.
 *
 * Since these apply to all null-owned groups, this allows anyone to update their group to
 * the null owner if they want to take advantage of this decentralized approval system.
 *
 * Currently, the affected transaction types are:
 * - AddGroupAdminTransaction
 * - RemoveGroupAdminTransaction
 *
 * This same approach could ultimately be applied to other group transactions too.
2022-09-19 11:03:06 +01:00
CalDescent
5017072f6c Use path parameter instead of query string. 2022-09-17 13:50:04 +01:00
CalDescent
02ac6dd8c1 Added GET /chat/message/{signature} endpoint.
This will ease the transition to a Q-Chat protocol, where chat messages will no longer be regular transactions.
2022-09-17 13:28:32 +01:00
CalDescent
858269f6cb ChatTransaction MAX_DATA_SIZE increased from 256 to 1024 bytes, to allow for new UI features. 2022-09-17 12:21:56 +01:00
CalDescent
791a9b78ec Added support for Pirate Chain wallets on FreeBSD. 2022-09-17 10:36:25 +01:00
CalDescent
aff49e6bdf Added support for ARRR refunds via /crosschain/htlc/refund/{ataddress} and /crosschain/htlc/refundAll
This could probably be refactored into multiple classes to make the code cleaner, but it is functional for now.
2022-09-17 10:30:10 +01:00
CalDescent
2d29fdca00 Allow BTC trades in redeemAll / refundAll, since most will now be using ACCTv3. 2022-09-16 11:19:10 +01:00
CalDescent
063ef8507b Fix for NPE in last commit 2022-09-11 20:04:51 +01:00
CalDescent
f042b5ca5f If mempow is active, remove any legacy accounts from a to-be-minted block that are missing a nonce. 2022-09-11 11:41:11 +01:00
CalDescent
a10e669554 Allow nonce to be computed for "next" timestamp if mempow is enabled in settings. 2022-09-11 11:36:19 +01:00
CalDescent
501f66ab00 BlockTransformer updates necessary for mempow online accounts.
Using the feature trigger timestamp here should be much less error prone than a whole new block message version. Once mempow has been live for at least 24 hours, the feature trigger can be removed and the code cleaned up, as all online accounts signatures will use the new format from that time onwards (legacy signatures are trimmed after 24 hours).
2022-09-10 18:31:01 +01:00
CalDescent
6003ed3ff7 Revert "Use block's online accounts timestamp (instead of main timestamp) for the mempow hard fork."
This reverts commit 8cca6db31637dcf43cbcf524949f629c3b35b6a6.
2022-09-10 17:23:39 +01:00
CalDescent
03e3619817 Revert "Use onlineAccountTimestamp for all mempow hard fork related code in OnlineAccountsManager too."
This reverts commit 23423102e70561471cd42db4ac035624d33f0bd0.
2022-09-10 17:23:34 +01:00
CalDescent
0e42e7b05a Merge branch 'master' into online-accounts-mempow-v2-block-updates
# Conflicts:
#	src/test/resources/test-chain-v2-no-sig-agg.json
2022-09-10 13:50:26 +01:00
CalDescent
d4fbc1687b Optionally exclude initial data from all trade websockets, using query string parameter excludeInitialData=true
Due to the large amount of data, it can take some time for the request to be processed and data to be transferred. It may make more sense to load the initial state from the standard API, and just use the websockets for updates.
2022-09-09 18:45:51 +01:00
DrewMPeacock
1abceada20 Fix up CREATE_POLL and VOTE_ON_POLL transactions to process and validate.
Added rule to enforce that a poll creator is also its owner.
2022-09-09 11:20:46 -06:00
CalDescent
8ffdc9b369 POST /crosschain/htlc/importarchivedtrades moved to POST /admin/repository/importarchivedtrades, as this is a repository operation not an HTLC one. 2022-09-09 18:01:20 +01:00
CalDescent
c883dd44c8 Remove LitecoinACCTv2 as it is no longer being used, and is unsafe. 2022-09-09 17:40:10 +01:00
CalDescent
667530e202 Remove DogecoinACCTv2 as it is no longer being used, and is unsafe. 2022-09-09 17:12:44 +01:00
CalDescent
5807d6e0dc Merge branch 'shares-by-level-rework' 2022-09-09 11:35:56 +01:00
CalDescent
ba4eeed358 Modified GET /arbitrary/resources endpoint (and underlying db queries) to allow filtering names by a list, e.g. "followedNames" or "blockedNames". 2022-09-07 18:42:24 +01:00
CalDescent
82edc4d9f3 OnlineAccountsV3Message MIN_PEER_VERSION set to 3.5.1 2022-09-07 18:26:30 +01:00
CalDescent
2a0d5746e6 Only compute "next" online account signature if mempow hard fork is active.
This minimizes the amount of differences in the first phase of the mempow rollout.
2022-09-04 13:19:32 +01:00
CalDescent
23423102e7 Use onlineAccountTimestamp for all mempow hard fork related code in OnlineAccountsManager too. 2022-09-04 11:51:24 +01:00
CalDescent
8879ec5bb4 Avoid computing the proof of work for each online account more than once. 2022-09-03 17:11:13 +01:00
CalDescent
8cca6db316 Use block's online accounts timestamp (instead of main timestamp) for the mempow hard fork.
This ensures that we cleanly switch from the old to new online account format, even if a block is minted retroactively.
2022-09-03 16:30:03 +01:00
CalDescent
effe1ac44d Merge branch 'master' into online-accounts-mempow-v2-block-updates
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
#	src/main/resources/blockchain.json
#	src/test/resources/test-chain-v2-no-sig-agg.json
2022-09-03 15:16:01 +01:00
CalDescent
ad4308afdf Added POST /crosschain/htlc/importarchivedtrades API endpoint.
This imports all trades from TradeBotStatesArchive.json into the repository.
2022-09-02 19:12:36 +01:00
CalDescent
6cfd85bdce Skip over ARRR orders in /refundAll and /redeemAll, as ARRR support hasn't been added for these yet.
This should fix 500 error which could prevent forced refund attempts for LTC and other chains. It wouldn't have affected any normal trade-bot refund functionality.
2022-09-02 18:05:12 +01:00
CalDescent
8b61247712 Added shareBinsByLevelV2.
This allows for different share bin distribution starting at an undecided future block height. This height will correspond with the QORA reduction. New values decided in recent community vote.
2022-09-02 16:43:46 +01:00
CalDescent
a9267760eb qoraHoldersShare reworked to qoraHoldersShareByHeight.
This allows the QORA share percentage to be modified at different heights, based on community votes. Added unit test to simulate a reduction.

# Conflicts:
#	src/test/java/org/qortal/test/minting/RewardTests.java
2022-08-29 09:40:20 +01:00
CalDescent
73396490ba Set walletsPath and listsPath to AppData folder for new Windows installs. 2022-08-27 19:44:31 +01:00
CalDescent
0b8fcc0a7b Bump version to 3.5.0 2022-08-26 19:32:58 +01:00
CalDescent
3d3ecbfb15 Merge branch 'master' into pirate-chain 2022-08-23 09:51:15 +01:00
CalDescent
9658f0cdd4 Revert "Rewrite of isNotOldPeer predicate, to fix logic issue."
This rewrite may have been causing problems with connections in the network, due to peers being forgotten too easily. Reverting for now to see if it solves the problem.

This reverts commit d81071f2544ec72807f70892388ae2ddec8dbc8c.

# Conflicts:
#	src/main/java/org/qortal/network/Network.java
2022-08-21 12:18:40 +01:00
CalDescent
b23500fdd0 Attempt to hold peer connections for 1-4 hours instead of 5-60 mins, as the constant disconnections are causing too much data to be sent over and over.
If this proves to not have any significant bad effects on re-orgs, we could consider setting these even higher or even disabling the auto disconnect by default.
2022-08-21 12:16:44 +01:00
CalDescent
a1365e57d8 Don't log network stats if no messages have been received, as otherwise this floods the logs with empty stats due to failed connections. 2022-08-21 12:14:12 +01:00
CalDescent
d8ca3a455d
Merge pull request #93 from catbref/peer-message-stats
Log count & total size of peer messages sent & received when a peer is disconnected
2022-08-21 11:41:37 +01:00
CalDescent
dcc943a906 Hopeful fix for InvalidKeyException seen in some JDK implementations. 2022-08-20 15:33:52 +01:00
CalDescent
cd2010bd06 More algo logging. 2022-08-20 12:50:24 +01:00
CalDescent
8cd16792a2 More logging relating to decryption failures. 2022-08-20 12:45:20 +01:00
CalDescent
4d97586f82 Log exception if AES decryption fails using specific algorithm settings. 2022-08-20 12:27:17 +01:00
CalDescent
3612fd8257 Removed unused throw 2022-08-20 11:58:16 +01:00
CalDescent
ff96868bd9 Log full stack trace if loading a QDN resource fails due to a DataException. 2022-08-20 11:57:53 +01:00
CalDescent
1694d4552e ArbitraryDataReader.deleteWorkingDirectory() is now optional. 2022-08-20 11:57:04 +01:00
CalDescent
bb1593efd2 Log full IOException stacktrace when obtaining Pirate Chain libraries. 2022-08-20 09:47:37 +01:00
CalDescent
4140546afb Ignore failures when deleting original QDN compressed file. 2022-08-20 09:45:48 +01:00
CalDescent
19197812d3 Log DataException during transaction validation. 2022-08-19 22:54:52 +01:00
CalDescent
168d32a474 Include memo for outgoing ARRR transactions. 2022-08-17 19:41:43 +01:00
CalDescent
a4fade0157 Validate wallet initialization result when restoring existing wallet. 2022-08-17 19:27:13 +01:00
CalDescent
2ea6921b66 Added more ARRR lightwalletd nodes 2022-08-17 19:26:47 +01:00
CalDescent
11ef31215b Fixed failing test 2022-08-17 19:25:45 +01:00
CalDescent
830a608b14 Include memo for incoming ARRR transactions. 2022-08-17 19:23:54 +01:00
CalDescent
57acf7dffe Updated text when downloading wallet files from QDN 2022-08-15 20:09:16 +01:00
CalDescent
9debebe03e Default birthday for ARRR moved to "arrrDefaultBirthday" (default 2000000).
This allows users to increase their default birthday if they know that no wallets were created before a certain block, to reduce sync time. It also fixed some failed unit tests that relied on transactions between blocks 1900000 and 2000000.
2022-08-13 16:48:03 +01:00
catbref
b17e96e121 Log count & total size of peer messages sent & received when a peer is disconnected. Requires org.qortal.net.Peer logging level set to DEBUG 2022-08-13 15:42:24 +01:00
CalDescent
b46c3cf95f Allow direct connection QDN retries every hour, instead of every 24 hours. 2022-08-13 14:20:10 +01:00
CalDescent
86526507a6 Increase time range and total number of attempts to fetch a QDN resource, as it previously gave up too quickly. 2022-08-12 22:25:35 +01:00
CalDescent
1b9128289f Added initial support to download Pirate wallet libraries from QDN, using hardcoded transaction signature.
Using a hardcoded signature ensures that the libraries cannot be swapped out without a core auto update, which requires the standard dev team approval process.
2022-08-12 21:57:31 +01:00
CalDescent
4a58f90223 Added support for multiple lightwalletd servers. 2022-08-12 21:52:02 +01:00
CalDescent
e68db40d91 Remove sapling params 2022-08-12 21:50:41 +01:00
CalDescent
bd6c0c9a7d qdata utility renamed to qdn 2022-08-12 13:38:30 +01:00
CalDescent
5804b9469c Default qdata port set to 12391 2022-08-12 13:38:03 +01:00
DrewMPeacock
4c463f65b7 Add API handles to build CREATE_POLL and VOTE_ON_POLL transactions. 2022-08-08 15:58:46 -06:00
CalDescent
53b47023ac ARRR default birthday increased to 2000000 2022-08-07 19:15:19 +01:00
CalDescent
22f9f08885 Added Windows amd64 architecture (again with a temporary path) 2022-08-07 19:14:21 +01:00
CalDescent
f26267e572 Don't sync, save, or load null seed wallets. They only act as a placeholder wallet when redeeming/refunding a P2SH. 2022-08-07 18:38:27 +01:00
CalDescent
e8c29226a1 Merge branch 'master' into pirate-chain 2022-08-06 16:41:18 +01:00
CalDescent
94f48f8f54 Don't allow QORT to be listed on the ARRR market unless the Pirate light wallet library is loaded. 2022-08-06 11:48:51 +01:00
CalDescent
3aac580f2c Add amd64 architecture (using a temporary path) 2022-08-06 11:47:36 +01:00
CalDescent
2d0b035f98 Updated AdvancedInstaller project for v3.4.3 2022-08-01 19:58:19 +01:00
CalDescent
075385d3ff Bump version to 3.4.3 2022-07-31 19:50:31 +01:00
CalDescent
6ed8250301 onlineAccountsModulusV2Timestamp set to Sat, 06 Aug 2022 16:00:00 UTC 2022-07-31 19:50:05 +01:00
CalDescent
d10ff49dcb Replaced arm architecture with aarch64, as 32 bit is unsupported. 2022-07-31 19:43:49 +01:00
CalDescent
4cf34fa932 Merge branch 'master' into pirate-chain 2022-07-31 16:00:52 +01:00
CalDescent
06b5d5f1d0 Merge branch 'increase-online-timestamp-modulus' 2022-07-31 13:00:47 +01:00
CalDescent
d6d2641cad Added "bitcoinjLookaheadSize" setting (default 50).
This replaces the WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ constant.
2022-07-31 12:07:44 +01:00
CalDescent
e71f22fd2c Added "gapLimit" setting.
This replaces the previously hardcoded "numberOfAdditionalBatchesToSearch" variable, and specifies the minimum number of empty consecutive addresses required before a set of wallet transactions is considered complete. Used for foreign transaction lists and balances.
2022-07-31 11:59:14 +01:00
CalDescent
c996633732 Added trace level logging. 2022-07-30 19:06:04 +01:00
CalDescent
55f973af3c Ensure all online accounts timestamps are a multiple of the online timestamp modulus.
This is a simple way to discard the 5-minute online account timestamps (from out of date nodes) once the switch to 30-minute online account timestamps has taken place.
2022-07-30 18:15:16 +01:00
CalDescent
fe9744eec6 Fixed missing feature trigger in testchain config 2022-07-30 14:52:47 +01:00
CalDescent
410fa59430 Merge branch 'master' into increase-online-timestamp-modulus
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
2022-07-30 12:23:25 +01:00
CalDescent
522ae2bce7 Updated AdvancedInstaller project for v3.4.2 2022-07-27 22:52:48 +01:00
CalDescent
a6e79947b8 Bump version to 3.4.2 2022-07-23 18:07:35 +01:00
CalDescent
b9bf945fd8 Removed aggregateSignatureTimestamp. All online account signatures are aggregated - there is no need for backwards support as signatures are trimmed from blocks after 24 hours. testOnlineAccountsModulusV2() had to be removed as this relied on pre-aggregation signatures. 2022-07-22 13:57:40 +01:00
CalDescent
85a27c14b8 Revert incorrect genesis timestamp that somehow made it into the stashed code. 2022-07-20 10:38:58 +01:00
CalDescent
46c40ca9ca Committed stashed code that is functional but probably too messy for production use.
Online account nonces are appended to the onlineAccountsSignatures to avoid the need for a new field, but it probably makes more sense to separate them.
2022-07-20 10:27:09 +01:00
CalDescent
fcd0d71cb6 Merge branch 'share-bin-activation' 2022-07-17 19:20:19 +01:00
catbref
275bee62d9 Revert BlockMinter to using long-lifetime repository session.
Although BlockMinter could reattach a repository session to its cache of potential blocks,
and these blocks would in turn reattach that repository session to their transactions,
further transaction-specific fields (e.g. creator PublicKeyAccount) were not being updated.

This would lead to NPEs like the following:

Exception in thread "BlockMinter" java.lang.NullPointerException
        at org.qortal.repository.hsqldb.HSQLDBRepository.cachePreparedStatement(HSQLDBRepository.java:587)
        at org.qortal.repository.hsqldb.HSQLDBRepository.prepareStatement(HSQLDBRepository.java:569)
        at org.qortal.repository.hsqldb.HSQLDBRepository.checkedExecute(HSQLDBRepository.java:609)
        at org.qortal.repository.hsqldb.HSQLDBAccountRepository.getBalance(HSQLDBAccountRepository.java:327)
        at org.qortal.account.Account.getConfirmedBalance(Account.java:72)
        at org.qortal.transaction.MessageTransaction.isValid(MessageTransaction.java:200)
        at org.qortal.block.Block.areTransactionsValid(Block.java:1190)
        at org.qortal.block.Block.isValid(Block.java:1137)
        at org.qortal.controller.BlockMinter.run(BlockMinter.java:301)

where the Account has an associated repository session which is now obsolete.

This commit reverts BlockMinter back to obtaining a repository session before entering main loop.
2022-07-17 13:53:07 +01:00
CalDescent
97221a4449 Added test to simulate level 7-8 reward tier activation, including orphaning. 2022-07-17 13:37:52 +01:00
CalDescent
508a34684b Revert "qoraHoldersShare reworked to qoraHoldersShareByHeight."
This reverts commit 90e8cfc737bf38f8104277a3aa0e1323dfcfdf64.

# Conflicts:
#	src/test/java/org/qortal/test/minting/RewardTests.java
2022-07-16 18:45:49 +01:00
CalDescent
3d2144f303 Check orphaning in levels 7-8 and 9-10 reward tests. This would have been tested in orphanCheck() anyway, but this makes the testing a bit more granular. 2022-07-16 17:29:16 +01:00
CalDescent
3c7fbed709 Fixed build error due to merge. 2022-07-15 11:53:30 +01:00
CalDescent
fb9a155e4c Merge branch 'master' into pirate-chain
# Conflicts:
#	src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java
#	src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java
2022-07-15 11:46:45 +01:00
CalDescent
fbcc870d36 Added informational test to compare ConsiceSet size against an int array for online account nonce arrays. 2022-07-15 10:23:14 +01:00
CalDescent
020e59743b Fixed failing test(s) due to merge. 2022-07-10 19:49:24 +01:00
CalDescent
0904de3f71 Merge branch 'master' into online-accounts-mempow-v2
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
2022-07-10 16:50:28 +01:00
CalDescent
35f3430687 Added share bin activation feature.
To prevent a single or very small number of minters receiving the rewards for an entire tier, share bins can now require "activation". This adds the requirement that a minimum number of accounts must be present in a share bin before it is considered active. When inactive, the rewards and minters are added to the previous tier.

Summary of new functionality:

- If a share bin has more than one, but less than 30 accounts present, the rewards and accounts are shifted to the previous share bin.
- This process is iterative, so the accounts can shift through multiple tiers until the minimum number of accounts is met, OR the share bin's starting level is less than shareBinActivationMinLevel.
- Applies to level 7+, so that no backwards support is needed. It will only take effect once the first account reaches level 7.

This requires hot swapping the sharesByLevel data to combine tiers where needed, so is a considerable shift away from the immutable array that was in place previously.

All existing and new unit tests are now passing, however a lot more testing will be needed.
2022-07-10 12:09:44 +01:00
CalDescent
90e8cfc737 qoraHoldersShare reworked to qoraHoldersShareByHeight.
This allows the QORA share percentage to be modified at different heights, based on community votes. Added unit test to simulate a reduction.
2022-07-08 11:12:58 +01:00
CalDescent
57bd3c3459 Merge remote-tracking branch 'catbref/auto-update-fix' 2022-07-07 18:48:39 +01:00
CalDescent
ad0d8fac91 Bump version to 3.4.1 2022-07-05 20:56:40 +01:00
CalDescent
a8b58d2007 Reward share limit activation timestamp set to 1657382400000 (Sat Jul 09 2022 16:00:00 UTC) 2022-07-05 20:34:23 +01:00
CalDescent
a099ecf55b Merge branch 'reduce-reward-shares' 2022-07-04 19:58:48 +01:00
CalDescent
6b91b0477d Added version query string param to /blocks/signature/{signature}/data API endpoint, to allow for optional V2 block serialization (with a single combined AT states hash).
Version can only be specified when querying unarchived blocks; archived blocks require V1 for now (and possibly V2 in the future).
2022-07-04 19:57:54 +01:00
CalDescent
fe2c63e8e4 Generate random nonces for test accounts.
These don't have to be valid for unit tests, because they are treated as "cached already valid accounts" in the block validation.
2022-07-02 17:30:31 +01:00
CalDescent
a3febdf00e Pass timestamp to OnlineAccountsManager.isMemoryPoWActive() so that block timestamp can be used. 2022-07-02 17:26:53 +01:00
CalDescent
4ca174fa0b Fixed bug in isMemoryPoWActive() which affected the ability to enable mempow via settings. 2022-07-02 13:45:39 +01:00
CalDescent
294582f136 Added mempow support in OnlineAccountsManager.
- This adds mempow requirements to online account importing (after activation timestamp), however doesn't yet add any requirements to block validation.
- It also causes the 'next' online accounts timestamp to be computed in addition to the 'current', so that the computed nonce value is ready when the next online accounts timestamp window begins.
2022-07-02 12:33:02 +01:00
CalDescent
d7e7c1f48c Fixed bugs from merge conflict, causing incorrect systray statuses in some cases. 2022-07-02 10:33:00 +01:00
CalDescent
215800fb67 Added optional "timeout" parameter to MemoryPoW.compute2().
This can be used to give up after the specified number of milliseconds.
2022-07-01 22:36:51 +01:00
CalDescent
b05d428b2e Added onlineAccountsMemPoWEnabled setting (for beta testing) 2022-07-01 22:31:51 +01:00
CalDescent
d2adadb600 Added onlineAccountsMemoryPoWTimestamp to blockchain.json 2022-07-01 22:31:30 +01:00
CalDescent
8e8c0b3fc5 Added OnlineAccountsV3Message, along with optional nonce Integer in OnlineAccountData.
This could potentially be released ahead of the other mempow code, splitting the rollout into multiple smaller phases.
2022-07-01 22:29:05 +01:00
CalDescent
65d63487f3 Merge branch 'master' into increase-online-timestamp-modulus 2022-07-01 17:46:35 +01:00
CalDescent
7c5932a512 GET /admin/status now returns online account submission status for "isMintingPossible", instead of BlockMinter status.
Online account credit is a more useful definition of "minting" than block signing, from the user's perspective. Should bring UI minting/syncing status in line with the core's systray status.
2022-07-01 17:29:15 +01:00
CalDescent
610a3fcf83 Improved order in getNodeType() 2022-07-01 16:48:57 +01:00
CalDescent
b329dc41bc Updated incorrect ONLINE_ACCOUNTS_V3_PEER_VERSION to 3.4.0 2022-07-01 13:36:56 +01:00
CalDescent
ff78606153 Merge branch 'master' into increase-online-timestamp-modulus 2022-07-01 13:14:15 +01:00
CalDescent
ef249066cd Updated another reference of SimpleTransaction::getTimestamp 2022-07-01 13:13:55 +01:00
CalDescent
80188629df Don't aggregate signatures when running OnlineAccountsTests, as it's too difficult to check how many unique signatures exist over a given period of time. 2022-07-01 13:13:22 +01:00
CalDescent
f77093731c Merge branch 'master' into increase-online-timestamp-modulus
# Conflicts:
#	src/main/java/org/qortal/block/Block.java
#	src/main/java/org/qortal/controller/OnlineAccountsManager.java
2022-07-01 13:03:19 +01:00
CalDescent
ca7d58c272 SimpleTransaction.timestamp is now in milliseconds instead of seconds.
Should fix 1970 timestamp issue in UI for foreign transactions, and also maintains consistency with QORT wallet transactions.
2022-07-01 12:46:20 +01:00
CalDescent
08f3351a7a Reward share transaction modifications:
- Reduce concurrent reward share limit from 6 to 3 (or from 5 to 2 when including self share) - as per community vote.
- Founders remain at 6 (5 when including self share) - also decided in community vote.
- When all slots are being filled, require that at least one is a self share, so that not all can be used for sponsorship.
- Activates at future undecided timestamp.
2022-07-01 12:18:48 +01:00
QuickMythril
f499ada94c
Merge pull request #91 from qortish/master
Update SysTray_sv.properties
2022-06-29 08:44:47 -04:00
qortish
f073040c06 Update SysTray_sv.properties
proper
2022-06-29 14:04:27 +02:00
CalDescent
49bfb43bd2 Updated AdvancedInstaller project for v3.4.0 2022-06-28 22:56:11 +01:00
CalDescent
425c70719c Bump version to 3.4.0 2022-06-28 19:29:09 +01:00
CalDescent
1420aea600 aggregateSignatureTimestamp set to 1656864000000 (Sun Jul 03 2022 16:00:00 UTC) 2022-06-28 19:26:00 +01:00
CalDescent
4543062700 Updated blockchain.json files in unit tests to include an already active "aggregateSignatureTimestamp" 2022-06-28 19:22:54 +01:00
CalDescent
722468a859 Restrict relays to v3.4.0 peers and above, in attempt to avoid bugs causing older peers to break relay chains. 2022-06-27 19:38:30 +01:00
CalDescent
492a9ed3cf Fixed more message rebroadcasts that were missing IDs. 2022-06-26 20:02:08 +01:00
CalDescent
420b577606 No longer adding inferior chain signatures in comparePeers() as it doesn't seem 100% reliable in some cases. It's better to re-check weights on each pass. 2022-06-26 18:24:33 +01:00
CalDescent
434038fd12 Reduced online accounts log spam 2022-06-26 16:34:04 +01:00
CalDescent
a9b154b783 Modified BlockMinter.higherWeightChainExists() so that it checks for invalid blocks before treating a chain as higher weight. Otherwise minting is slowed down when a higher weight but invalid chain exists on the network (e.g. after a hard fork). 2022-06-26 15:54:41 +01:00
CalDescent
a01652b816 Removed hasInvalidBlock filtering, as this was unnecessary risk now that the original bug in comparePeers() is fixed. 2022-06-26 10:09:24 +01:00
CalDescent
4440e82bb9 Fixed long term bug in comparePeers() causing peers with invalid blocks to prevent alternate valid but lower weight candidates from being chosen. 2022-06-25 16:34:42 +01:00
CalDescent
a2e1efab90 Synchronize hasInvalidBlock predicate, as it wasn't thread safe 2022-06-25 14:12:21 +01:00
CalDescent
7e1ce38f0a Fixed major bug in hasInvalidBlock predicate 2022-06-25 14:11:25 +01:00
CalDescent
a93bae616e Invalid signatures are now stored as ByteArray instead of String, to avoid regular Base58 encoding and decoding, which is very inefficient. 2022-06-25 13:29:53 +01:00
CalDescent
a2568936a0 Synchronizer: filter out peers reporting to hold invalid block signatures.
We already mark peers as misbehaved if they returned invalid signatures, but this wasn't sufficient when multiple copies of the same invalid block exist on the network (e.g. after a hard fork). In these cases, we need to be more proactive to avoid syncing with these peers, to increase the chances of preserving other candidate blocks.
2022-06-25 12:45:19 +01:00
CalDescent
23408827b3 Merge remote-tracking branch 'catbref/schnorr-agg-BlockMinter-fix' into schnorr-agg-BlockMinter-fix
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
#	src/main/java/org/qortal/controller/OnlineAccountsManager.java
#	src/main/java/org/qortal/network/message/BlockV2Message.java
#	src/main/resources/blockchain.json
#	src/test/resources/test-chain-v2.json
2022-06-24 11:47:58 +01:00
CalDescent
ae6e2fab6f Rewrite of isNotOldPeer predicate, to fix logic issue (second attempt - first had too many issues)
Previously, a peer would be continuously considered not 'old' if it had a connection attempt in the past day. This prevented some peers from being removed, causing nodes to hold a large repository of peers. On slower systems, this large number of known peers resulted in low numbers of outbound connections being made, presumably because of the time taken to iterate through dataset, using up a lot of allKnownPeers lock time.

On devices that experienced the problem, it could be solved by deleting all known peers. This adds confidence that the old peers were the problem.
2022-06-24 10:36:06 +01:00
CalDescent
3af36644c0 Revert "Rewrite of isNotOldPeer predicate, to fix logic issue."
This reverts commit d81071f2544ec72807f70892388ae2ddec8dbc8c.
2022-06-24 10:26:39 +01:00
CalDescent
db8f627f1a Default minPeerVersion set to 3.3.7 2022-06-24 10:13:55 +01:00
CalDescent
5db0fa080b Prune peers every 5 minutes instead of every cycle of the Controller thread.
This should reduce the amount of time the allKnownPeers lock is held.
2022-06-24 10:13:36 +01:00
CalDescent
d81071f254 Rewrite of isNotOldPeer predicate, to fix logic issue.
Previously, a peer would be continuously considered not 'old' if it had a connection attempt in the past day. This prevented some peers from being removed, causing nodes to hold a large repository of peers. On slower systems, this large number of known peers resulted in low numbers of outbound connections being made, presumably because of the time taken to iterate through dataset, using up a lot of allKnownPeers lock time.

On devices that experienced the problem, it could be solved by deleting all known peers. This adds confidence that the old peers were the problem.
2022-06-24 10:11:46 +01:00
QuickMythril
ba148dfd88
Added Korean translations
credit: TL (Discord username)
2022-06-23 02:35:42 -04:00
QuickMythril
ff40b8f8ab Updated German translations 2022-06-23 01:43:33 -04:00
CalDescent
dbcb457a04 Merge branch 'master' of github.com:Qortal/qortal 2022-06-20 22:51:33 +01:00
CalDescent
b00e1c8f47 Allow online account submission in all cases when in recovery mode. 2022-06-20 22:50:41 +01:00
CalDescent
899a6eb104 Rework of systray statuses
- Show "Minting" as long as online accounts are submitted to the network (previously it related to block signing).
- Fixed bug causing it to regularly show "Synchronizing 100%".
- Only show "Synchronizing" if the chain falls more than 2 hours behind - anything less is unnecessary noise.
2022-06-20 22:48:32 +01:00
CalDescent
6e556c82a3 Updated AdvancedInstaller project for v3.3.7 2022-06-20 22:25:40 +01:00
CalDescent
35ce64cc3a Bump version to 3.3.7 2022-06-20 21:52:41 +01:00
CalDescent
09b218d16c Merge branch 'master' of github.com:Qortal/qortal 2022-06-20 21:51:39 +01:00
CalDescent
7ea451e027 Allow trades to be initiated, and QDN data to be published, as long as the latest block is within 60 minutes of the current time. Again this should remove negative effects of larger re-orgs from the UX. 2022-06-20 21:26:48 +01:00
CalDescent
ffb27c3946 Further relaxed min latest block timestamp age to be considered "up to date" in a few places, from 30 to 60 mins. This should help reduce the visible effects of larger re-orgs if they happen again. 2022-06-20 21:25:14 +01:00
QuickMythril
6e7d2b50a0
Added Romanian translations
credit: Ovidiu (Telegram username)
2022-06-20 01:27:28 -04:00
QuickMythril
bd025f30ff
Updated ApiError German translations
credit: CD (Discord username)
2022-06-20 00:56:19 -04:00
CalDescent
c6cbd8e826 Revert "Sync behaviour changes:"
This reverts commit 8a76c6c0de1992c0694df409eb551782c5c1b6a0.
2022-06-19 19:10:37 +01:00
CalDescent
b85afe3ca7 Revert "Keep existing findCommonBlocksWithPeers() and comparePeers() behaviour prior to consensus switchover, to reduce the number of variables."
This reverts commit fecfac5ad94b38c99dde3413750ed0c69ec45bff.
2022-06-19 19:10:30 +01:00
CalDescent
5a4674c973 Revert "newConsensusTimestamp set to Sun Jun 19 2022 16:00:00 UTC"
This reverts commit 55a0c1085506e0d75896366e0abc1c09e4381376.
2022-06-19 19:09:57 +01:00
CalDescent
769418e5ae Revert "Fixed unit tests due to missing feature trigger"
This reverts commit 28f9df7178b6ecc39d9116669192e0b68fabd597.
2022-06-19 19:09:52 +01:00
CalDescent
38faed5799 Don't submit online accounts if node is more than 2 hours of sync. 2022-06-19 18:56:08 +01:00
catbref
10a578428b Improve intermittent auto-update failures, mostly under non-Windows environments.
Symptoms are:
* AutoUpdate trying to run new ApplyUpdate process, but nothing appears in log-apply-update.?.txt
* Main qortal.jar process continues to run without updating
* Last AutoUpdate line in log.txt.? is:
	2022-06-18 15:42:46 INFO  AutoUpdate:258 - Applying update with: /usr/local/openjdk11/bin/java -Djava.net.preferIPv4Stack=false -Xss256k -Xmx1024m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=127.0.0.1:5005 -cp new-qortal.jar org.qortal.ApplyUpdate

Changes are:
* child process now inherits parent's stdout / stderr (was piped from parent)
* child process is given a fresh stdin, which is immediately closed
* AutoUpdate now converts -agentlib JVM arg to -DQORTAL_agentlib
* ApplyUpdate converts -DQORTAL_agentlib to -agentlib

The latter two changes are to prevent a conflict where two processes try to reuse the same JVM debugging port number.
2022-06-19 18:25:00 +01:00
CalDescent
96cdf4a87e Updated AdvancedInstaller project for v3.3.6 2022-06-19 11:41:50 +01:00
CalDescent
c0b1580561 Bump version to 3.3.6 2022-06-17 20:42:51 +01:00
CalDescent
28f9df7178 Fixed unit tests due to missing feature trigger 2022-06-17 13:07:33 +01:00
CalDescent
55a0c10855 newConsensusTimestamp set to Sun Jun 19 2022 16:00:00 UTC 2022-06-17 13:04:04 +01:00
CalDescent
7c5165763d Merge branch 'sync-long-tip' 2022-06-17 13:02:08 +01:00
CalDescent
d2836ebcb9 Make sure to set the ID when rebroadcasting messages. Hopeful fix for QDN networking issues. 2022-06-16 22:39:51 +01:00
CalDescent
fecfac5ad9 Keep existing findCommonBlocksWithPeers() and comparePeers() behaviour prior to consensus switchover, to reduce the number of variables. 2022-06-16 18:38:27 +01:00
CalDescent
5ed1ec8809 Merge remote-tracking branch 'catbref/sync-long-tip' into sync-long-tip 2022-06-16 18:35:16 +01:00
catbref
431cbf01af BlockMinter will discard block candidates that turn out to be invalid just prior to adding transactions, to be potentially reminted in the next pass 2022-06-16 17:47:08 +01:00
CalDescent
af792dfc06 Updated AdvancedInstaller project for v3.3.5 2022-06-15 21:11:12 +01:00
CalDescent
d3b6c5f052 Bump version to 3.3.5 2022-06-14 23:18:46 +01:00
CalDescent
f48eb27f00 Revert "Temporarily limit block minter as first stage of multipart minting fix. This will be reverted almost immediately after release."
This reverts commit 0eebfe4a8c07cf2ed984ee03ed7c18d7b900064b.
2022-06-14 22:42:44 +01:00
CalDescent
b02ac2561f Revert "Safety check - also to be removed shortly."
This reverts commit 8b3f9db497189faaf933d49d18a972549ba32547.
2022-06-14 22:42:38 +01:00
CalDescent
1b2f66b201 Bump version to 3.3.4 2022-06-13 23:40:39 +01:00
CalDescent
e992f6b683 Increase column size to allow for approx 10x as many online accounts in each block. 2022-06-13 22:51:48 +01:00
CalDescent
8b3f9db497 Safety check - also to be removed shortly. 2022-06-13 22:41:10 +01:00
CalDescent
0eebfe4a8c Temporarily limit block minter as first stage of multipart minting fix. This will be reverted almost immediately after release. 2022-06-13 22:40:54 +01:00
lexandr0s
c03344caae
Update ApiError_ru.properties 2022-06-04 23:57:25 +04:00
lexandr0s
237b39a524
Update SysTray_ru.properties 2022-06-04 23:50:03 +04:00
CalDescent
12b3fc257b Updated AdvancedInstaller project for v3.3.3 2022-06-04 19:50:43 +01:00
CalDescent
66a3322ea6 Bump version to 3.3.3 2022-06-04 19:16:01 +01:00
CalDescent
4965cb7121 Set BlockV2Message min peer version to 3.3.3 2022-06-04 15:50:17 +01:00
CalDescent
b92b1fecb0 disableReferenceTimestamp set to 1655222400000 (Tuesday, 14 June 2022 16:00:00 UTC) 2022-06-04 14:51:37 +01:00
CalDescent
43a75420d0 Merge branch 'disable-reference' 2022-06-04 14:46:51 +01:00
catbref
e85026f866 Initial work on reducing network load for transferring blocks.
Reduced AT state info from per-AT address + state hash + fees to AT count + total AT fees + hash of all AT states.
Modified Block and Controller to support above. Controller needs more work regarding CachedBlockMessages.
Note that blocks fetched from archive are in old V1 format.
Changed Triple<BlockData, List<TransactionData>, List<ATStateData>> to BlockTransformation to support both V1 and V2 forms.

Set min peer version to 3.3.203 in BlockV2Message class.
2022-06-04 14:43:46 +01:00
CalDescent
ba7b9f3ad8 Added feature to allow repository to be kept intact after running certain tests 2022-06-04 14:17:01 +01:00
catbref
4eb58d3591 BlockTimestampTests to show results from changing blockTimingsByHeight 2022-06-04 12:36:36 +01:00
catbref
8d8e58a905 Network$NetworkProcessor now has its own LOGGER 2022-06-04 12:36:36 +01:00
catbref
8f58da4f52 OnlineAccountsManager:
Bump v3 min peer version from 3.2.203 to 3.3.203
No need for toOnlineAccountTimestamp(long) as we only ever use getCurrentOnlineAccountTimestamp().
Latter now returns Long and does the call to NTP.getTime() on behalf of caller, removing duplicated NTP.getTime() calls and null checks in multiple callers.
Add aggregate-signature feature-trigger timestamp threshold checks where needed, near sign() and verify() calls.
Improve logging - but some logging will need to be removed / reduced before merging.
2022-06-04 12:36:36 +01:00
catbref
a4e2aedde1 Remove debug-hindering "final" modifier from effectively final locals 2022-06-04 12:36:36 +01:00
catbref
24d04fe928 Block.mint() always uses latest timestamped online accounts 2022-06-04 12:36:35 +01:00
catbref
0cf32f6c5e BlockMinter now only acquires repository instance as needed to prevent long HSQLDB rollbacks 2022-06-04 12:35:54 +01:00
catbref
84d850ee0b WIP: use blockchain feature-trigger "aggregateSignatureTimestamp" to determine when online-accounts sigs and block sigs switch to aggregate sigs 2022-06-04 12:35:51 +01:00
catbref
51930d3ccf Move some private key methods to Crypto class 2022-06-04 12:35:15 +01:00
catbref
c5e5316f2e Schnorr public key and signature aggregation for 'online accounts'.
Aggregated signature should reduce block payload significantly,
as well as associated network, memory & CPU loads.

org.qortal.crypto.BouncyCastle25519 renamed to Qortal25519Extras.
Our class provides additional features such as DH-based shared secret,
aggregating public keys & signatures and sign/verify for aggregate use.

BouncyCastle's Ed25519 class copied in as BouncyCastleEd25519,
but with 'private' modifiers changed to 'protected',
to allow extension by our Qortal25519Extras class,
and to avoid lots of messy reflection-based calls.
2022-06-04 12:35:15 +01:00
catbref
829ab1eb37 Cherry-pick minor fixes from another branch to resolve "No online accounts - not even our own?" issues 2022-06-04 12:35:03 +01:00
catbref
d9b330b46a OnlineAccountData no longer uses signature in equals() or hashCode() because newer aggregate signatures use random nonces and OAD class doesn't care about / verify sigs 2022-06-04 10:49:59 +01:00
catbref
c032b92d0d Logging fix: size() was called on wrong collection, leading to confusing logging output 2022-06-04 10:49:59 +01:00
catbref
ae92a6eed4 OnlineAccountsV3: slightly rework Block.mint() so it doesn't need to filter so many online accounts
Slight optimization to BlockMinter by adding OnlineAccountsManager.hasOnlineAccounts():boolean instead of returning actual data, only to call isEmpty()!
2022-06-04 10:49:59 +01:00
catbref
712c4463f7 OnlineAccountsV3:
Move online account cache code from Block into OnlineAccountsManager, simplifying Block code and removing duplicated caches from Block also.
This tidies up those remaining set-based getters in OnlineAccountsManager.
No need for currentOnlineAccountsHashes's inner Map to be sorted so addAccounts() creates new ConcurentHashMap insteaad of ConcurrentSkipListMap.

Changed GetOnlineAccountsV3Message to use a single byte for count of hashes as it can only be 1 to 256.
256 is represented by 0.

Comments tidy-up.
Change v3 broadcast interval from 10s to 15s.
2022-06-04 10:49:59 +01:00
catbref
fbdc1e1cdb OnlineAccountsV3:
Adding support for GET_ONLINE_ACCOUNTS_V3 to Controller, which calls OnlineAccountsManager.

With OnlineAccountsV3, instead of nodes sending their list of known online accounts (public keys),
nodes now send a summary which contains hashes of known online accounts, one per timestamp + leading-byte combo.
Thus outgoing messages are much smaller and scale better with more users.
Remote peers compare the hashes and send back lists of online accounts (for that timestamp + leading-byte combo) where hashes do not match.

Massive rewrite of OnlineAccountsManager to maintain online accounts.
Now there are three caches:
1. all online accounts, but split into sets by timestamp
2. 'hashes' of all online accounts, one hash per timestamp+leading-byte combination
Mainly for efficient use by GetOnlineAccountsV3 message constructor.
3. online accounts for the highest blocks on our chain to speed up block processing
Note that highest blocks might be way older than 'current' blocks if we're somewhat behind in syncing.

Other OnlineAccountsManager changes:
* Use scheduling executor service to manage subtasks
* Switch from 'synchronized' to 'concurrent' collections
* Generally switch from Lists to Sets - requires improved OnlineAccountData.hashCode() - further work needed
* Only send V3 messages to peers with version >= 3.2.203 (for testing)
* More info on which online accounts lists are returned depending on use-cases

To test, change your peer's version (in pom.xml?) to v3.2.203.
2022-06-04 10:49:59 +01:00
catbref
f2060fe7a1 Initial work on online-accounts-v3 network messages to drastically reduce network load.
Lots of TODOs to action.
2022-06-04 10:49:59 +01:00
catbref
6950c6bf69 Initial work on reducing network load for transferring blocks.
Reduced AT state info from per-AT address + state hash + fees to AT count + total AT fees + hash of all AT states.
Modified Block and Controller to support above. Controller needs more work regarding CachedBlockMessages.
Note that blocks fetched from archive are in old V1 format.
Changed Triple<BlockData, List<TransactionData>, List<ATStateData>> to BlockTransformation to support both V1 and V2 forms.

Set min peer version to 3.3.203 in BlockV2Message class.
2022-06-04 10:49:11 +01:00
catbref
8a76c6c0de Sync behaviour changes:
* Prefer longer chains
* Compare chain tips
* Skip pre-sync all-peer sorting (for now)
2022-06-04 10:23:31 +01:00
CalDescent
ef51cf5702 Added defensiveness in getOnlineTimestampModulus(), just in case NTP.getTime() returns null 2022-06-02 12:46:11 +01:00
CalDescent
0c3988202e Merge branch 'master' into increase-online-timestamp-modulus 2022-06-02 12:38:05 +01:00
CalDescent
987446cf7f Updated AdvancedInstaller project for v3.3.2 2022-06-02 11:31:56 +01:00
CalDescent
6dd44317c4 Sync pirate wallet every 30 seconds instead of 60, to match behaviour of official wallet. 2022-05-31 10:09:37 +02:00
CalDescent
d2fc705846 Merge remote-tracking branch 'catbref/UnsupportedMessage' 2022-05-31 09:09:08 +02:00
CalDescent
e393150e9c Require references to be the correct length post feature-trigger 2022-05-30 22:40:39 +02:00
CalDescent
43bfd28bcd Revert "Discard unsupported messages instead of disconnecting the peer."
This reverts commit d086ade91f3afd0a72c10ed9b395980884474dad.
2022-05-30 21:46:16 +02:00
catbref
ca8f8a59f4 Better forwards compatibility with newer message types so we don't disconnect newer peers 2022-05-30 20:41:45 +01:00
CalDescent
85a26ae052 Another rework of null seed wallets, to allow them to be saved and loaded.
A full sync is unavoidable for P2SH redeem/refund, so we need to be able to save our progress. Creating a new null seed wallet each time isn't an option because it relies on having a recent checkpoint to avoid having to sync large amounts of blocks every time (sync is per wallet, not per node).
2022-05-30 19:31:55 +02:00
CalDescent
c30b1145a1 Improved ensureSynchronized() as it would often not notice an unsynced wallet. 2022-05-30 18:14:26 +02:00
CalDescent
d086ade91f Discard unsupported messages instead of disconnecting the peer. 2022-05-30 15:27:42 +02:00
CalDescent
64d4c458ec Fixed logging error 2022-05-30 15:16:44 +02:00
CalDescent
2478450694 Revert "Removed "consecutive blocks" limitation in block minter."
This reverts commit f41fbb3b3d9b4b7f6ec44686fdd1f69ede08fb3e.
2022-05-30 13:49:15 +02:00
CalDescent
9f19a042e6 Added test to ensure short (1 byte) references can be imported. 2022-05-30 13:46:00 +02:00
CalDescent
922ffcc0be Modified post-trigger last reference checking, to now require a non-null value
This allows for compatibility with TRANSFER_PRIVS validation in commit 8950bb7, which treats any account with a non-null reference as "existing". It also avoids possible unknown side effects from trying to process and store transactions with a null reference - something that wouldn't have been possible until the validation was removed.
2022-05-30 13:22:15 +02:00
CalDescent
f887fcafe3 Disable last reference validation after feature trigger timestamp (not yet set).
This should prevent the failed transactions that are encountered when issuing two or more in a short space of time. Using a feature trigger (hard fork) to release this, to avoid potential consensus confusion around the time of the update (older versions could consider the main chain invalid until updating).
2022-05-30 13:06:55 +02:00
CalDescent
48b562f71b Auto update check interval slowed from 10s to 30s, to hopefully reduce the chance of encountering "repository in use by another process?" error. 2022-05-28 14:33:37 +02:00
CalDescent
5203742b05 Started work on architecture-specific lite wallet library loading. Paths are not yet correct. 2022-05-28 14:31:38 +02:00
CalDescent
f14b494bfc "Disposable" wallets renamed to "null seed" wallets, as this is a better description of what they are. 2022-05-28 14:28:42 +02:00
CalDescent
9a4ce57001 Increase blockchain lock wait time from 30 to 60 seconds in /transactions/process.
This will hopefully reduce the number of failed tradeoffer listings that result in a nonfunctional tradebot (and subsequent PENDING status shown in the UI)
2022-05-28 13:33:09 +02:00
CalDescent
10af961fdf Consider a node with a block in the last 30 mins to be "up to date" when trading. 2022-05-27 22:31:21 +02:00
CalDescent
33cffe45fd Bump version to 3.3.2 2022-05-27 17:23:53 +02:00
CalDescent
a0ce75a978 Minimum BTC order amount set to 0.001 BTC. Anything lower than that will result in greater than 10% fees. 2022-05-27 16:59:34 +02:00
QuickMythril
8d168f6ad4
Override default Bitcoin trade fee
Reduced to 20 sats/byte.
2022-05-27 06:57:38 -04:00
CalDescent
0875c5bf3b Fix ConcurrentModificationException in getCachedSigValidTransactions() 2022-05-27 10:34:26 +02:00
CalDescent
b17b28d9d6 Catch NoSuchMethodError in ElectrumX, and log it, just in case we ever reencounter a dependency issue. 2022-05-27 10:33:37 +02:00
CalDescent
e95249dc1b Reduced bouncycastle version to 1.69, as 1.70 was having compatibility issues with the ElectrumX code. 2022-05-27 10:30:28 +02:00
CalDescent
bb4bdfede5 Added concept of a "disposable" pirate chain wallet.
This is needed to allow redeem/refund of P2SH without having an actively synced and initialized wallet. It also ultimately avoids us having to retain the wallet entropy in the trade bot states. Various safety checks have been introduced to make sure that a disposable wallet is never used for anything other than P2SH redeem/refund.
2022-05-27 10:29:24 +02:00
CalDescent
e2b241d416 Fix ConcurrentModificationException in getCachedSigValidTransactions() 2022-05-27 10:15:41 +02:00
CalDescent
aeb94fb879 Merge branch 'master' into pirate-chain 2022-05-27 09:16:38 +02:00
QuickMythril
8e71cbd822
Reduce static Bitcoin trade fee 2022-05-26 16:25:07 -04:00
QuickMythril
9896ec2ba6
Fix typo 2022-05-26 15:37:38 -04:00
QuickMythril
9f9a74809e Add Bitcoin ACCTv3
This provides support for restoring BTC in the Trade Portal.
2022-05-26 15:34:51 -04:00
QuickMythril
acce81cdcd
Add tray menu item to show Build Version
Core build version in a message dialog for OS which cannot display the entire tooltip.
2022-05-26 15:10:04 -04:00
CalDescent
d72953ae78 Drop expired transactions from the import queue before they are considered "sig valid".
This should prevent expired transactions from being kept alive, adding unnecessary load to the import queue.
2022-05-25 19:06:08 +01:00
CalDescent
32213b1236 Catch all exceptions in PirateLightClient 2022-05-24 20:48:57 +01:00
CalDescent
761d461bad Bump bouncycastle version to 1.70, necessary for ALPN support in some JDKs. 2022-05-24 20:35:19 +01:00
CalDescent
774a3b3dcd Catch RuntimeException, so that the gRPC client is shutdown. 2022-05-24 19:52:23 +01:00
CalDescent
30567d0e87 Correctly handle shutdown of gRPC managed channel on error. 2022-05-24 19:32:41 +01:00
CalDescent
6b53eb5384 Pirate Chain uses the 'b' prefix for P2SH addresses, but the light wallet library is configured to use 't3' (from Zcash), so it's easiest to just derive a different prefix for each destination.
This could be simplified by configuring the light wallet library to use the correct 'b' prefix, but this didn't work when first attempted.
2022-05-23 23:14:54 +01:00
CalDescent
767ef62b64 Added PirateChain.isValidWalletKey() 2022-05-23 22:11:23 +01:00
CalDescent
f7e6d1e5c8 Merge branch 'master' into pirate-chain 2022-05-23 21:55:49 +01:00
CalDescent
551686c2de Updated AdvancedInstaller project for v3.3.1 2022-05-23 21:54:25 +01:00
CalDescent
b73c041cc3 Bump version to 3.3.1 2022-05-23 20:31:36 +01:00
CalDescent
9e8d85285f Removed extra unnecessary digest after writing new data. 2022-05-22 16:13:08 +01:00
CalDescent
f41fbb3b3d Removed "consecutive blocks" limitation in block minter. 2022-05-22 16:13:04 +01:00
CalDescent
3f5240157e Return more detailed errors in trade portal APIs. 2022-05-22 16:11:20 +01:00
CalDescent
7c807f754e First draft of Pirate Chain trade bot. Not tested yet.
This has been modified to a) use full public keys instead of PKH, and b) hand off all transaction building, signing, and broadcasting to the (heavily customized) Pirate light wallet library.
2022-05-22 16:07:48 +01:00
CalDescent
9e1b23caf6 Removed unused constants. 2022-05-22 16:01:12 +01:00
CalDescent
c2bad62d36 Updated status text when not initialized. 2022-05-22 16:00:37 +01:00
CalDescent
4516d44cc0 UnspentOutput additions to support latest PirateChainHTLC methods 2022-05-22 15:55:40 +01:00
CalDescent
9c02b01318 Added PirateChainHTLC.getUnspentFundingTxid(), allowing a minimum amount to be specified.
This will ensure that the correct fundingTxid can be redeemed or refunded by the trade bot.
2022-05-22 15:55:01 +01:00
CalDescent
08fab451d2 Added PirateChainHTLC.getFundingTxid(), to lookup the txid that funded a P2SH. 2022-05-22 15:49:14 +01:00
CalDescent
d47570c642 First draft of Pirate Chain ACCT, with modifications to allow for 33 byte public keys instead of 20 byte PKH. Pirate Chain HTLCs are required to use full public keys rather than hashes. 2022-05-22 12:23:01 +01:00
CalDescent
4547386b1f Increase receiving_account_info column size from 32 to 128 bytes, to allow for Pirate Chain sapling shielded addresses, which are much longer. 2022-05-21 15:24:37 +01:00
CalDescent
ab01dc5e54 Implemented address validation for Pirate Chain 2022-05-21 09:22:49 +01:00
CalDescent
380c742aad Pirate chain minimum order size temporarily decreased to 0.0001, for testing only. 2022-05-21 09:11:24 +01:00
CalDescent
368359917b Use LegacyZcashAddress (copied and modified from bitcoinj) to derive and encode Pirate Chain P2SH addresses.
It wasn't possible to use bitcoinj directly because Zcash-style P2SH addresses have 2 prefix characters (t3), which isn't supported by bitcoinj.
2022-05-20 21:27:38 +01:00
CalDescent
1c1b570cb3 Added Pirate TLC and raw transaction tests 2022-05-20 21:22:18 +01:00
CalDescent
3fe43372a7 Added PirateChainHTLC - uses a different HTLC format suitable for Pirate Chain.
Also removes transaction building code, since this is handled by the light wallet library.
2022-05-20 21:21:51 +01:00
CalDescent
c7bc1d7dcd Hardcode fee to MAINNET_FEE when sending coins. 2022-05-20 21:16:12 +01:00
CalDescent
a7ea6ec80d Added wrapper methods for Pirate LiteWalletJni P2SH funding/redeeming/refunding.
These require a heavily customized version of both piratewallet-light-cli and librustzcash.
2022-05-20 21:14:50 +01:00
CalDescent
9cf574b9e5 Added support for Pirate Chain transaction lookups and deserialization, necessary for HTLC status checks. 2022-05-20 21:07:54 +01:00
CalDescent
20e63a1190 Removed "consecutive blocks" limitation in block minter. 2022-05-20 13:38:56 +01:00
CalDescent
f6fc5de520 Removed extra unnecessary digest after writing new data. 2022-05-20 13:36:56 +01:00
CalDescent
0b89118cd1 Fixed bug in Pirate wallet transaction parsing. 2022-05-20 13:35:52 +01:00
CalDescent
fa3a81575a Reduce wasted time that could otherwise be spent validating queued transaction signatures. 2022-05-14 12:53:36 +01:00
CalDescent
6990766f75 Speed up unconfirmed transaction propagation.
Currently, new transactions take a very long time to be included in each block (or reach the intended recipient), because each node has to obtain a repository lock and import the transaction before it notifies its peers. This can take a long time due to the lock being held by the block minter or synchronizer, and this compounds with every peer that the transaction is routed through.

Validating signatures doesn't require a lock, and so can take place very soon after receipt of a new transaction. This change causes each node to broadcast a new transaction to its peers as soon as its signature is validated, rather than waiting until after the import.

When a notified peer then makes a request for the transaction data itself, this can now be loaded from the sig-valid import queue as an alternative to the repository (since they won't be in the repository until after the import, which likely won't have happened yet).

One small downside to this approach is that each unconfirmed transaction is now notified twice - once after the signature is deemed valid, and again in Controller.onNewTransaction(), but this should be an acceptable trade off given the speed improvements it should achieve. Another downside is that it could cause invalid transactions (with valid signatures) to propagate, but these would quickly be added to each peer's invalidUnconfirmedTransactions list after the import failure, and therefore be ignored.
2022-05-14 12:43:54 +01:00
CalDescent
e1e1a66a0b Improvements and fixes to PirateWallet 2022-05-14 12:10:35 +01:00
CalDescent
e552994f68 Merge branch 'master' into pirate-chain 2022-05-13 13:23:26 +01:00
CalDescent
b33afd99a5 Bump invalid transaction import logging from trace to debug. 2022-05-13 13:22:58 +01:00
CalDescent
3c2ba4a0ea Improved logging when importing transactions. 2022-05-13 12:31:48 +01:00
CalDescent
ab0fc07ee9 Refactored transaction importer, to separate signature validation from importing.
Importing has to be single threaded since it requires the database lock, but there's nothing to stop us from validating signatures on multiple threads, as no lock is required. So it makes sense to separate these two functions to allow for possible multi threaded signature validation in the future, to speed up the process.

Everything remains single threaded in this commit. It should be functionally the same as before, to reduce risk.
2022-05-13 12:31:19 +01:00
CalDescent
001650d48e Fixed typo in currently unused getSignaturesInvolvingAddress() method. 2022-05-13 11:28:55 +01:00
CalDescent
659431ebfd Added "txTypes" parameter to GET /transactions/unconfirmed, to allow optional filtering of unconfirmed transactions by one or more types 2022-05-13 11:28:21 +01:00
CalDescent
0a419cb105 Added "creator" parameter to GET /transactions/unconfirmed, to allow filtering unconfirmed transactions by creator's public key 2022-05-13 11:13:39 +01:00
CalDescent
a4d4d17b82 Added /wallets to .gitignore, in preparation for pirate chain support. 2022-05-12 19:34:53 +01:00
CalDescent
0829ff6908 Fixed bugs in tooltip, introduced in lite node branch 2022-05-12 19:33:12 +01:00
CalDescent
2d1b0fd6d0 Merge branch 'lite-node' 2022-05-12 08:51:05 +01:00
CalDescent
122539596d Merge branch 'qdn-direct-connections' 2022-05-12 08:45:45 +01:00
CalDescent
86015e59a1 Updated AdvancedInstaller project for v3.3.0 2022-05-10 08:27:17 +01:00
CalDescent
107a23f1ec Switch to PirateChainMainNetParams 2022-05-10 08:05:32 +01:00
CalDescent
abce068b97 Catch UnsatisfiedLinkError when initializing Pirate Chain library 2022-05-10 08:05:06 +01:00
CalDescent
28fd9241d4 Fixed issues with merge 2022-05-10 08:01:07 +01:00
CalDescent
3fc4746a52 Merge branch 'master' into pirate-chain
# Conflicts:
#	src/test/java/org/qortal/test/crosschain/DigibyteTests.java
#	src/test/java/org/qortal/test/crosschain/RavencoinTests.java
2022-05-10 07:54:47 +01:00
CalDescent
1ea1e00344 Bump version to 3.3.0 2022-05-09 18:47:19 +01:00
CalDescent
598f219105 Don't require a full synchronization in order to return the ARRR wallet address. 2022-05-08 08:26:00 +01:00
CalDescent
bbf7193c51 Fixed bugs in wallet initialization 2022-05-08 08:24:54 +01:00
CalDescent
adecb21ada Added mainnet xpub addresses for DGB and RVN tests, as testnet support isn't fully implemented yet. 2022-05-07 17:40:36 +01:00
CalDescent
fa4679dcc4 Updated DGB and RVN tests due to conflict. 2022-05-07 17:05:10 +01:00
CalDescent
58917eeeb4 "walletsPath" is now configurable in the settings. 2022-05-07 17:01:19 +01:00
CalDescent
f36e193650 Merge branch 'master' into pirate-chain
# Conflicts:
#	src/main/java/org/qortal/settings/Settings.java
2022-05-07 16:50:32 +01:00
CalDescent
dac484136f Fixed bug in name rebuilding. 2022-05-07 16:46:10 +01:00
CalDescent
999ad857ae Fixed typo 2022-05-07 16:33:10 +01:00
CalDescent
d073b9da65 Added support for Pirate Chain wallets.
Note: this relies on (a modified version of) liblitewallet-jni which is not included, but will ultimately be compiled for each supported architecture and hosted on QDN.

LiteWalletJni code is based on https://github.com/PirateNetwork/cordova-plugin-litewallet - thanks to @CryptoForge for the help in getting this up and running.
2022-05-07 16:32:04 +01:00
CalDescent
aaa0b25106 Make sure to set Peer.isDataPeer() to false as well as true, to prevent bugs due to object reuse.
Also designate a peer as a "data peer" when making an outbound connection to request data from it.
2022-05-02 10:20:23 +01:00
CalDescent
f7dabcaeb0 Increase ONLINE_ACCOUNTS_MODULUS from 5 to 30 mins at a future undecided timestamp.
Note: it's important that this timestamp is set on a 1-hour boundary (such as 16:00:00) to ensure a clean switchover.

# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
2022-05-02 08:50:08 +01:00
CalDescent
3409086978 Merge branch 'master' into lite-node 2022-05-02 08:31:43 +01:00
CalDescent
6c201db3dd Merge branch 'EPC-fixes' 2022-05-02 08:28:14 +01:00
CalDescent
da47df0a25 Fixed merge issue. 2022-05-01 16:41:56 +01:00
CalDescent
eea215dacf Merge branch 'ravencoin' into new-coins
# Conflicts:
#	pom.xml
#	src/main/java/org/qortal/controller/tradebot/TradeBot.java
#	src/main/java/org/qortal/crosschain/SupportedBlockchain.java
#	src/main/java/org/qortal/settings/Settings.java
2022-05-01 16:28:22 +01:00
CalDescent
0949271dda Merge branch 'digibyte' into new-coins 2022-05-01 16:25:40 +01:00
CalDescent
6bb9227159 Removed RavencoinACCTv1
Also removed CrossChainRavencoinACCTv1Resource - same as Digibyte.
2022-05-01 16:17:00 +01:00
CalDescent
a95a37277c Removed DigibyteACCTv1 and v2
Also removed CrossChainDigibyteACCTv1Resource, since this is unused, and it seems excessive to maintain support of this for every coin (and potentially every ACCT version).
2022-05-01 16:12:12 +01:00
CalDescent
48b9aa5c18 Allow images to be displayed in QDN websites via data: and blob: 2022-05-01 14:37:57 +01:00
CalDescent
1d7203a6fb Bug fixes found when testing previous commits. 2022-05-01 14:29:24 +01:00
CalDescent
1030b00f0a Keep track of peers requesting data for which we have at least one chunk. Then allow subsequent incoming connections from that peer through, up to a maximum of maxDataPeers.
Direct connections for arbitrary data are currently unlikely to succeed, because those allowing incoming connections generally have their slots maxed out and have reached maxPeers. The idea here is that some connections remain reserved for dedicated arbitrary data transfers, therefore temporarily circumventing the limit (up to a defined maximum number of reserved connections).

Arbitrary data connections will auto disconnect after 2 minutes (we might be able to reduce this at a later date), and it also probably makes sense for the requesting node to disconnect as soon as it has all the chunks that it needs (this part isn't implemented yet).

One downside of this feature is that the listen socket is now going to be accepting connections most of the time, since it is unlikely that we will regularly have 4 data peers connected. This could be improved by modifying the OP_ACCEPT behaviour based on whether we are expecting any data peers to connect. In most cases, this would allow it to remain closed. But for the sake of simplicity I will leave that optimization for a future commit.
2022-05-01 14:02:44 +01:00
CalDescent
0c16d1fc11 Added "maxDataPeerConnectionTime" setting (default 2 mins).
This is used to force a quick disconnect for peers that are only connecting for the purposes of requesting data for a specific arbitrary transaction signature.
2022-05-01 14:02:44 +01:00
CalDescent
ed04375385 Increased default maxPeers from 32 to 36 to compensate - otherwise the network will lose a considerable amount of inbound capacity. 2022-05-01 14:02:44 +01:00
CalDescent
6e49d20383 Added "maxDataPeers" setting to reserve 4 connections by default for direct QDN data requests. 2022-05-01 14:02:44 +01:00
CalDescent
dc34eed203 Include our address when requesting QDN data 2022-05-01 14:02:44 +01:00
CalDescent
fbe4f3fad8 Fixed incorrect minOutboundPeers conditional 2022-05-01 14:01:58 +01:00
CalDescent
e7ee3a06c7 Merge branch 'EPC-fixes' into lite-node 2022-04-30 15:33:07 +01:00
CalDescent
599877195b Merge branch 'master' into EPC-fixes 2022-04-30 15:32:44 +01:00
CalDescent
7f9d267992 Improved lite node response error logging. 2022-04-30 15:32:23 +01:00
CalDescent
52904db413 Migrated new lite node message types to new format. 2022-04-30 15:22:50 +01:00
CalDescent
5e0bde226a Merge branch 'EPC-fixes' into lite-node
# Conflicts:
#	src/main/java/org/qortal/network/message/Message.java
2022-04-30 13:25:02 +01:00
CalDescent
0695039ee3 Fixed long term bug causing last line to be missed out. 2022-04-30 12:08:10 +01:00
CalDescent
a4bcd4451c Added "tail" parameter to GET /admin/logs to allow returning the last X (limit) lines.
This should make it easy to display core logs in the UI.
2022-04-30 12:07:47 +01:00
CalDescent
e5b4b61832 Fixed bugs causing "Hash ... does not match file digest ..." errors 2022-04-30 11:26:05 +01:00
CalDescent
dd55dc277b Updated AdvancedInstaller project for v3.2.5 2022-04-27 08:50:40 +01:00
CalDescent
81ef1ae964 Bump version to 3.2.5 2022-04-26 20:17:57 +01:00
CalDescent
46701e4de7 Revert "Remove peers with unknown height, lower height or same height and same block signature (unless we don't have their block signature)"
This reverts commit 895f02f17896c5ad78f7e8f26252d7530aae3a85.
2022-04-26 19:52:08 +01:00
QuickMythril
0f52ccb433 add Ravencoin ACCTs 2022-04-26 13:51:19 -04:00
QuickMythril
8aed84e6af add Digibyte ACCTs 2022-04-26 11:40:42 -04:00
CalDescent
568497e1c5 Updated AdvancedInstaller project for v3.2.4 2022-04-25 09:03:57 +01:00
CalDescent
f3f8e0013d Bump version to 3.2.4 2022-04-24 17:48:22 +01:00
CalDescent
d03c145189 Added to testRegisterNameFeeIncrease() test to catch the recently detected bug. 2022-04-24 17:41:24 +01:00
CalDescent
682a5fde94 Revert "Attempt to fix core startup problems on some systems (GNOME Desktop?) by adding defensiveness to GUI elements."
This reverts commit 311f41c610b7d3388cef714979d9653508fb0c55.
2022-04-24 15:54:20 +01:00
CalDescent
cca5bac30a Fixed logic bug in name registration fee calculation. 2022-04-24 15:36:36 +01:00
CalDescent
64e102a8c6 Name registration fee reduction to 1.25 QORT set to Sun, 01 May 2022 16:00:00 GMT 2022-04-24 15:27:21 +01:00
CalDescent
f9972f50e0 Updated altcoinj 2022-04-24 15:08:43 +01:00
QuickMythril
05d9a7e820 Switched to Qortal fork of altcoinj, using RavencoinMainNetParams 2022-04-23 08:28:12 -04:00
CalDescent
df290950ea Reduce log spam in BlockMinter 2022-04-23 12:32:06 +01:00
CalDescent
ae64be4802 Retry scheduled repository maintenance up to 5 times, as it's common for it to timeout waiting for the repository. Subsequent retries normally succeed. 2022-04-23 12:31:15 +01:00
CalDescent
348f3c382e Merge branch 'master' into lite-node 2022-04-22 20:40:47 +01:00
CalDescent
d98678fc5f Renamed SECRET_LENGTH to SECRET_SIZE_LENGTH. Thanks to catbref for finding this. 2022-04-22 20:40:13 +01:00
CalDescent
1da157d33f Added separate AT serialization tests, based on generic transaction serialization tests. This allows for testing both MESSAGE-type and PAYMENT-type AT transactions. 2022-04-22 20:36:22 +01:00
CalDescent
de4f004a08 Bump to transaction version 6 at a future undecided timestamp. 2022-04-22 20:35:17 +01:00
CalDescent
522ef282c8 Added support for deserialization of MESSAGE-type AT transactions (requires transaction version 6) 2022-04-22 20:34:42 +01:00
CalDescent
b5522ea260 Added support for PAYMENT-type AT transactions in serialization tests 2022-04-22 20:06:37 +01:00
CalDescent
b1f184c493 Use DigibyteMainNetParams 2022-04-22 16:31:44 +01:00
CalDescent
d66dd51bf6 Switched to Qortal fork of altcoinj, with Digibyte support 2022-04-22 16:31:32 +01:00
QuickMythril
0baed55a44 add DGB wallet 2022-04-21 11:40:17 -04:00
QuickMythril
390b359761 add RVN wallet 2022-04-21 11:38:49 -04:00
CalDescent
311f41c610 Attempt to fix core startup problems on some systems (GNOME Desktop?) by adding defensiveness to GUI elements. 2022-04-20 08:41:37 +01:00
CalDescent
0a156c76a2 Fix for NPE observed on the EPC-fixes branch (but putting the fix on master in case unrelated) 2022-04-20 08:38:59 +01:00
CalDescent
70eaaa9e3b Merge remote-tracking branch 'catbref/EPC-fixes' into EPC-fixes 2022-04-19 20:50:32 +01:00
catbref
3e622f7185 EPC-fixes: catch CancelledKeyExceptions thrown in short window between nextSelectionKey.isValid() and nextSelectionKey.isXXXable() calls 2022-04-18 14:33:05 +01:00
CalDescent
3f12be50ac Merge remote-tracking branch 'catbref/EPC-fixes' into EPC-fixes 2022-04-18 09:20:55 +01:00
catbref
68412b49a1 EPC-fixes: use bindAddress from Settings for outgoing peer connections, not just listen socket 2022-04-17 19:38:50 +01:00
catbref
c9b2620461 EPC-fixes: fix constructing GET_ONLINE_ACCOUNTS_V2 message for case where onlineAccount args is empty list 2022-04-17 19:37:28 +01:00
CalDescent
337b03aa68 Catch java.util.ServiceConfigurationError in Gui.loadImage() 2022-04-17 17:59:29 +01:00
catbref
df3f16ccf1 EPC-fixes: Improve Network shutdown by exiting fast during broadcast and skipping callbacks during peer disconnect. 2022-04-17 14:50:15 +01:00
catbref
22aa5c41b5 WIP: EPC-fixes
BlockMessage was broken because the repository 'connection' associated with the message's Block object was closed between message queuing and message sending.

The fix was to serialize Message subclasses on construction, thus freeing reliance on objects passed into constructor.
The serialized byte[] is held by the message between queuing and sending.
This forces messages into one of two 'modes': outgoing or incoming.
Outgoing messages contain serialized byte[] whereas incoming messages unpack a ByteBuffer into Message subclass fields.
As a result, all network message types have been refactored in this way.
More details in Message's class comment.

A knock-on effect is that incoming messages cannot then be sent out - a new message needs to be constructed.
Some changes needed to Arbitrary controller package classes in this respect.

Bonus: Network no longer needs broadcast threads because 'broadcasting' is now simply the act of queuing a message for many peers.
2022-04-17 14:50:15 +01:00
catbref
8e09567221 EPC-fixed: avoiding some CancelledKeyExceptions 2022-04-17 14:50:15 +01:00
catbref
3505788d42 Another chunk of improvements to networking / EPC.
Instead of synchronizing/blocking in Peer.sendMessage(),
we queue messages in a concurrent blocking TransferQueue, with timeout.

In EPC, ChannelWriteTasks consume from TransferQueue, unblocking callers to Peer.sendMessage().

If a new message is to be sent, or socket output buffer is full,
then OP_WRITE is used to wait for socket to become writable again.

Only one ChannelWriteTask per peer can be active/pending at a time.
Each ChannelWriteTask tries to send as much as it can in one go.

Other minor tidy-ups.
2022-04-17 14:50:15 +01:00
catbref
91e0c9b940 More improvements to networking:
As per work done by szisti in PR#45:
Extracted network 'Tasks' to their own classes.
Network.NetworkProcessor reduced to only producing Tasks.

Improved usage of SocketChannel interest-ops.
Eventually this might lead to reducing task-producing synchronization lock into more granular locks.
Work still needed to convert sending messages to a queue and to make use of OP_WRITE instead of sleeping to wait for socket buffer to empty.

Disabled PeerConnectTask producer from checking against connected peers via DNS as it's too slow.

Swapped Peer's replyQueues from SynchronizedMap(wrapped HashMap) to ConcurrentHashMap.

Other minor changes within networking.
2022-04-17 14:50:15 +01:00
catbref
00996b047f Networking work-in-progress:
As per work done by szisti in PR#45:
Extracted MessageException from inside Message into its own class.
Extracted MessageType from inside Message into its own class.

Converted reflection-based Message.fromByteBuffer method call to non-reflection, functional interface, method-reference.
This should have minor performance improvement but stronger method signature and type enforcement, as well as better IDE integration.

Message.fromByteBuffer method 'contract' tightened up to:
1. throw BufferUnderflowException if there are not enough bytes to deserialize message
2. throw MessageException if bytes contain invalid data
3. should not return null

Message.toData method 'contract' tightened up to:
1. return null if the message has no payload to serialize
2. throw IOException directly - no need to try-catch in each subclass

Several Message-subclass fields now marked 'final' as per IDE suggestion.
Several Message-subclass fromByteBuffer() method signatures have changed 'throws' list.
Several bytes.remaining() != some-value changed to bytes.remaining() < some-value as per new contract.

Some bytes.remaining() checks removed for fixed-length messages because we can rely on ByteBuffer throwing BufferUnderflowException.
Some bytes.remaining() checks retained for variable-length messages, or messages that read a large amount of data, to prevent wasted memory allocations.

Other minor tidying up
2022-04-17 14:50:15 +01:00
catbref
44fc0f367d Networking work-in-progress:
Temporarily increase sleep from 1ms to 100ms when waiting for outgoing socket buffer to empty.

Real fix is to rewrite using an outgoing message queue and OP_WRITE interest op.
2022-04-17 14:50:15 +01:00
catbref
b0e6259073 Networking work-in-progress:
De-register a peer's socket channel OP_READ interest op when producing a ChannelTask for that peer.
This should prevent duplicate ChannelTasks for the same peer.

Re-register OP_READ once node has read from peer's channel.
2022-04-17 14:50:15 +01:00
catbref
6255b2a907 Networking work-in-progress:
When node has reached max connections, Network will ignore pending incoming connections by:
1. not calling accept()
2. de-registering OP_ACCEPT 'interest op' on the listen socket's channel

When a peer disconnects, Network might re-register OP_ACCEPT interest op on listen socket.
2022-04-17 14:50:15 +01:00
catbref
a5fb0be274 Fix Network.disconnectPeer(PeerAddress) to prevent removeIf() on UnmodifiableList throwing UnsupportedOperationException 2022-04-17 14:50:15 +01:00
catbref
e835f6d998 ExecuteProduceConsume:
Slight reworking of EPC to simplify when producer can block
and generally make some of the conditional code more readable.

Improved logging with task class names and logging level editable during runtime!
Use /peer/enginestats?newLoggingLevel=DEBUG (or TRACE or back to INFO) to change.
2022-04-17 14:50:15 +01:00
CalDescent
54ff564bb1 Set name for transaction importer thread 2022-04-16 23:32:42 +01:00
CalDescent
f8a5ded0ba Fix for bug introduced in commit cfe9252 2022-04-16 23:20:49 +01:00
CalDescent
a1be66f02b Temporarily ease the filtering of lite node peers, in order to make development easier. 2022-04-16 20:52:31 +01:00
CalDescent
0815ad2cf0 Added AT transaction deserialization, to all them to be sent in messages for lite nodes.
Note that it is currently not easy to distinguish between MESSAGE-type and PAYMENT-type AT transactions, so PAYMENT-type is currently the only one supported (and used). A hard fork will likely be needed in order to specify the type within each message.
2022-04-16 20:52:31 +01:00
CalDescent
3484047ad4 Added GET /transactions/address/{address} API endpoint
This is a more standardized alternative to using GET /transactions/search?address=xyz. This avoids the need to build full transaction search ability into the lite node protocols right away.
2022-04-16 20:52:31 +01:00
CalDescent
a63fa1cce5 Added GET_ACCOUNT_TRANSACTIONS message, as well as a generic TRANSACTIONS message for responses. 2022-04-16 20:52:31 +01:00
CalDescent
59119ebc3b Added GET_NAME message to allow lookups from name to owner (or any other name data). 2022-04-16 20:52:31 +01:00
CalDescent
276f1b7e68 Fixed small errors in earlier commits. 2022-04-16 20:52:31 +01:00
CalDescent
c482e5b5ca Added GET_ACCOUNT_NAMES message to request names for an address, and a generic NAMES message to return a list of NameData objects. The generic NAMES message can be reused for many other responses, such as requesting the various lists of names that the API supports. 2022-04-16 20:52:31 +01:00
CalDescent
8c3e0adf35 Added message types to fetch account details and account balances, and use these in various APIs.
This should bring in enough data for very basic chat and wallet functionality (using addresses rather than registered names).

Data currently comes from a single random peer, however this can be expanded to request from multiple peers to gain confidence in the accuracy of the data. If bad data is returned from a peer, it's not the end of the world since the transaction would just be considered invalid by full nodes and would be thrown out. But this should be mostly avoidable by taking data from multiple sources to improve confidence in its accuracy.
2022-04-16 20:52:30 +01:00
CalDescent
64ff3ac672 Improved comment 2022-04-16 20:52:30 +01:00
CalDescent
cfe92525ed Disable various core functions when running as a lite node.
Lite nodes can't sync or mint blocks, and they also have a very limited ability to verify unconfirmed transactions due to a lack of contextual information (i.e. the blockchain). For now, most validation is skipped and they simply act as relays to help get transactions around the network. Full and topOnly nodes will disregard any invalid transactions upon receipt as usual, and since the lite nodes aren't signing any blocks, there is little risk to the reduced validation, other than the experience of the lite node itself. This can be tightened up considerably as the lite nodes become more powerful, but the current approach works as a PoC.
2022-04-16 20:52:30 +01:00
CalDescent
0e3a9ee2b2 Return the node "type" (full / topOnly / lite) in GET /admin/info endpoint.
This can used by the UI to hide features that aren't supported on lite nodes.
2022-04-16 20:50:27 +01:00
CalDescent
a921db2cc6 Added "lite" setting to designate the core as a lite node. 2022-04-16 20:50:27 +01:00
CalDescent
3d99f86630 Improved logging 2022-04-16 20:50:00 +01:00
CalDescent
8d1a58ec06 POW_DIFFICULTY_NO_QORT reduced from 14 to 12 (around 4x faster) 2022-04-16 12:36:32 +01:00
CalDescent
2e5a7cb5a1 Adapted Blockchain.java to use lookup table for name registration fees, to more easily support fee adjustments.
This is currently for name registration transactions only, but can be adapted (or duplicated) for other transaction types when needed.

Note: this switches from a greater-than (>) to a greater-than-or-equal (>=) timestamp comparison, as it makes more sense this way. It shouldn't affect the previous transition since there were no REGISTER_NAME transactions at that exact timestamp.
2022-04-16 12:20:03 +01:00
CalDescent
895f02f178 Remove peers with unknown height, lower height or same height and same block signature (unless we don't have their block signature)
Adapted from code originally written by catbref from before genesis, and essentially prevents syncing backwards. This needs significant testing on testnet.
2022-04-16 11:30:07 +01:00
CalDescent
c59869982b Fix for system-wide QDN issues occuring when the metadata file has an empty chunks array.
It is quite likely that existing resources with both metadata and an empty chunks array will need to be republished, because this bug may have led to incorrect file deletions.
2022-04-16 11:25:44 +01:00
CalDescent
3b3368f950
Merge pull request #85 from QuickMythril/member-count
Add member count to each group returned by GET /member/{address}
2022-04-16 11:00:35 +01:00
QuickMythril
3f02c760c2
Add member count to each group returned by GET /member/{address} 2022-04-15 06:23:10 -04:00
CalDescent
fee603e500 Add member count to each group returned by GET /groups (expanded on code written by QuickMythril) 2022-04-15 10:19:43 +01:00
QuickMythril
ad31d8014d get memberCount with Group Data
works for lookup by groupId
2022-04-14 22:08:52 -04:00
CalDescent
58a0ac74d2
Merge pull request #84 from catbref/ByteArray
Improvements to ByteArray to leverage Java 11 'native' Arrays methods
2022-04-14 21:30:59 +01:00
CalDescent
184984c16f Added ElectrumX equivalent for Pirate Chain, to communicate with a remote lightwalletd
This will most likely be used for by the trade bot, rather than for any wallet functionality.
2022-04-14 20:18:58 +01:00
CalDescent
2cf7a5e114 Generated java gRPC protocol buffers for Pirate Chain network comms.
The command used was:

./protoc --plugin=protoc-gen-grpc-java=/Users/user/Downloads/protoc-gen-grpc-java-1.45.1-osx-x86_64.exe -I=src/main/resources/proto/zcash/ --java_out=src/main/java/ --grpc-java_out=src/main/java/ src/main/resources/proto/zcash/service.proto

Then repeat, replacing service.proto with compact_formats.proto and darkside.proto

Darkside isn't needed for mainnet functionality, but included for completeness, and might be useful for testing.
2022-04-14 20:07:42 +01:00
QuickMythril
8388aa9c23 update Russian translation
credit: Alexander45 & malina
2022-04-10 15:50:29 -04:00
catbref
c1894d8c00 Improvements to ByteArray to leverage Java 11 'native' Arrays.hashCode and Arrays.compareUnsigned for speed.
Also modified ambiguous ByteArray::new and ByteArray::of to ByteArray::wrap and ByteArray::copyOf.
Modifications to other classes that use ByteArray.
2022-04-10 16:38:02 +01:00
QuickMythril
f7f9cdc518
Merge pull request #83 from aldum/feature/hungarian_translation
fixup grammar; add missing translations
2022-04-09 00:10:37 -04:00
QuickMythril
850d7f8220 add/update translations
credit: johnnyfg (sv), schizo (it), IsBe (nl), Eduardo9999 (es)
2022-04-08 23:57:54 -04:00
aldum
051043283c fixup grammar; add missing translations 2022-04-06 23:21:49 +02:00
QuickMythril
15bc69de01
Merge pull request #82 from JaymenChou/patch-8
Update SysTray_zh_CN.properties
2022-04-05 13:38:13 -04:00
QuickMythril
ee3cfa4d6d
fix typo 2022-04-05 13:26:02 -04:00
QuickMythril
df1f3079a5
Merge pull request #81 from JaymenChou/patch-7
Update SysTray_zh_TW.properties
2022-04-05 13:25:06 -04:00
QuickMythril
d9ae8a5552
Merge branch 'master' into patch-8 2022-04-05 13:23:19 -04:00
QuickMythril
2326c31ee7
Merge branch 'master' into patch-7 2022-04-05 13:11:14 -04:00
QuickMythril
91cb0f30dd
Updated TransactionValidity translations
added some missing entries, and sorted alphabetically.
2022-04-05 12:51:49 -04:00
QuickMythril
c0307c352c
Updated ApiError translations
removed some duplicate entries, and standardized the order
2022-04-05 11:46:32 -04:00
QuickMythril
8fd7c1b313 formatting fix 2022-04-05 11:09:30 -04:00
QuickMythril
b8147659b1
Updated SysTray translations
added some missing entries, and sorted alphabetically.
2022-04-05 10:48:43 -04:00
JaymenChou
7a1bac682f
Update SysTray_zh_TW.properties
Add the missing term "PERFORMING_DB_MAINTENANCE" and translate it to Traditional Chinese
2022-04-04 20:36:48 +08:00
JaymenChou
9fdb7c977f
Update SysTray_zh_CN.properties
Translate remaining terms to Simple Chinese
2022-04-04 20:33:59 +08:00
JaymenChou
4f3948323b
Update SysTray_zh_TW.properties
Translate the remaining terms to Traditional Chinese
2022-04-04 20:31:19 +08:00
QuickMythril
70fcc1f712
Merge pull request #78 from JaymenChou/patch-4
Create ApiError_zh_CN.properties
2022-04-04 02:49:00 -04:00
JaymenChou
f20fe9199f
Update ApiError_zh_CN.properties 2022-04-04 14:36:55 +08:00
QuickMythril
91dee4a3b8
Merge pull request #80 from JaymenChou/patch-6
Create TransactionValidity_zh_CN.properties
2022-04-04 02:17:35 -04:00
QuickMythril
0b89b8084e
Merge pull request #79 from JaymenChou/patch-5
Create TransactionValidity_zh_TW.properties
2022-04-04 02:17:24 -04:00
QuickMythril
a5a80302b2
Merge pull request #77 from JaymenChou/patch-3
Create ApiError_zh_TW.properties
2022-04-04 02:17:02 -04:00
QuickMythril
e61a24ee7b
removed electrum-ltc.bysh.me
this server often gives a false positive for phishing by some antivirus software.
2022-04-03 22:32:57 -04:00
JaymenChou
55ed342b59
Create TransactionValidity_zh_CN.properties
Add Simple Chinese For better understanding of logs
2022-04-03 13:27:52 +08:00
JaymenChou
3c6f79eec0
Create TransactionValidity_zh_TW.properties
Add Traditional Chinese For TransactionValidity Logs.
2022-04-03 13:25:32 +08:00
JaymenChou
590800ac1d
Create ApiError_zh_CN.properties
Add Simple Chinese Support For API Error Message 
Hope it helps in understanding the API !
2022-04-03 12:43:18 +08:00
JaymenChou
95c412b946
Create ApiError_zh_TW.properties
Add Traditional Chinese support to API Responses
2022-04-03 12:40:27 +08:00
CalDescent
a232395750 Merge branch 'master' of github.com:Qortal/qortal 2022-04-01 11:24:56 +01:00
QuickMythril
6edbc8b6a5
add decimal precision to download progress 2022-03-31 13:46:40 -04:00
QuickMythril
f8ffeed302
updated BTC electrumx servers
added new and removed TCP, closed servers, and versions older than 1.16.0
2022-03-31 11:32:55 -04:00
QuickMythril
e2ee68427c
removed TCP electrumx servers 2022-03-31 11:29:54 -04:00
QuickMythril
74ff23239d
removed TCP electrumx servers 2022-03-31 11:27:56 -04:00
QuickMythril
f1fa2ba2f6
added SSL electrumx servers 2022-03-31 10:02:31 -04:00
QuickMythril
e1522cec94
updated LTC electrumx servers 2022-03-31 09:58:53 -04:00
QuickMythril
8841b3cbb1
add spanish translations 2022-03-31 08:44:33 -04:00
CalDescent
94260bd93f Decreased the number of retries for missing metadata, to reduce broadcast spam. 2022-03-30 08:23:22 +01:00
CalDescent
15ff8af7ac Don't process trade bots or broadcast presence timestamps if our chain is more than 30 minutes old 2022-03-30 08:11:02 +01:00
CalDescent
d420033b36 Revert "Revert "Add Qortal AT FunctionCodes for getting account level / blocks minted + tests""
This reverts commit 59025b8f470374866af3477285c4c98838b06a45.
2022-03-30 08:07:07 +01:00
CalDescent
bda63f0310 Removed hardcoded "qortal-backup/TradeBotStates.json" from POST /admin/repository/data API, as it's no longer needed now that API keys are required. 2022-03-30 08:06:09 +01:00
QuickMythril
54add26ccb
fixed typo 2022-03-25 23:39:41 -04:00
CalDescent
089b068362 Updated AdvancedInstaller project for v3.2.3 2022-03-19 22:38:58 +00:00
CalDescent
fe474b4507 Bump version to 3.2.3 2022-03-19 20:44:41 +00:00
CalDescent
bbe15b563c Added unit test to simulate recent issue.
This fails with the 3.2.2 code but now passes when using the latest fixes.
2022-03-19 20:41:38 +00:00
CalDescent
59025b8f47 Revert "Add Qortal AT FunctionCodes for getting account level / blocks minted + tests"
This reverts commit eb9b94b9c6cb7f969cd0a759535be8e117a7359c.
2022-03-19 19:52:14 +00:00
CalDescent
1b42c5edb1 Fixed NPE in runIntegrityCheck()
This feature is disabled by default so can be tidied up later. For now, the unhandled scenario is logged and the checking continues on.

One name's transactions are too complex for the current integrity check code to verify (MangoSalsa), but it has been verified manually. All other names pass the automated test.
2022-03-19 19:22:16 +00:00
CalDescent
362335913d Fixed infinite loop in name rebuilding.
If an account is renamed and then at some point renamed back to one of the original names, it confused the names rebuilding code. The current solution is to track the linked names that have already been rebuilt, and then break out of the loop once a name is encountered a second time.
2022-03-19 18:55:19 +00:00
CalDescent
4340dac595 Fixed recently introduced issue in name rebuilding code causing transactions to be unordered.
This is the likely cause of inconsistent name entries across different nodes, as we can't guarantee that every environment will return the same transaction order from the SQL queries.
2022-03-19 18:44:16 +00:00
CalDescent
f3e1fc884c
Merge pull request #63 from catbref/master
Add Qortal AT FunctionCodes for getting account level / blocks minted
2022-03-19 11:32:39 +00:00
CalDescent
39c06d8817
Merge pull request #75 from catbref/name-unicode
Unicode / NAME updates.
2022-03-19 11:32:22 +00:00
CalDescent
91cee36c21 Catch and log all exceptions in addStatusToResources()
Some users are seeing 500 errors deriving from this code. This should hopefully allow more info to be obtained, as well as causing it to omit the status for resources that encounter problems.
2022-03-19 11:08:42 +00:00
CalDescent
6bef883942 Removed OpenJDK 11 reference in build-release.sh, as it seems that checksums will not match by default due to timestamps and file orderings.
See: https://dzone.com/articles/reproducible-builds-in-java
2022-03-19 11:05:51 +00:00
CalDescent
25ba2406c0 Updated AdvancedInstaller project for v3.2.2 2022-03-16 19:53:22 +00:00
CalDescent
e4dc8f85a7 Bump version to 3.2.2 2022-03-15 19:57:02 +00:00
CalDescent
12a4a260c8 Handle new sync result case. 2022-03-14 22:04:11 +00:00
CalDescent
268f02b5c3 Added automated test to ensure that the core's default bootstrap hosts are functional.
Whilst not strictly a unit test, this should allow issues with the core's bootstrap servers to be caught quickly.
2022-03-14 21:52:54 +00:00
CalDescent
13eff43b87 Fixed synchronizer issues which caused large re-orgs
Peers without a recent block are removed at the start of the sync process, however, due to the time lag involved in fetching block summaries and comparing the list of peers, some of these could subsequently drop back to a non-recent block and still be chosen as the next peer to sync with. The end result being that nodes could unnecessarily orphan as many as 20 blocks due to syncing with a peer that doesn't have a recent block (but has a couple of high weight blocks after the common block).

This commit adds some additional filtering to avoid this situation.

1) Peers without a recent block are removed as candidates in comparePeers(), allowing for alternate peers to be chosen.
2) After comparePeers() completes, the list is filtered a second time to make sure that all are still recent.
3) Finally, the peer's state is checked one last time in syncToPeerChain(), just before any orphaning takes place.

Whilst just one of the above would probably have been sufficient, the consequences of this bug are so severe that it makes sense to be very thorough.

The only exception to the above is when the node is in "recovery mode", in which case peers without recent blocks are allowed to be included. Items 1 and 3 above do not apply in recovery mode. Item 2 does apply, since the entire comparePeers() functionality is already skipped in a recovery situation due to our chain being out of date.
2022-03-14 21:47:37 +00:00
catbref
e604a19bce Unicode / NAME updates.
Fix UPDATE_NAME not processing empty 'newName' transactions correctly.
Fix some emoji code-points not being processed correctly.
Updated tests.
Now included ICU4J v70.1 - WARNING: this could add around 10MB to JAR size!
Bumped homoglyph to v1.2.1.
2022-03-14 08:45:32 +00:00
CalDescent
e63e39fe9a Updated AdvancedInstaller project for v3.2.1 2022-03-13 19:39:58 +00:00
CalDescent
584c951824 Bump version to 3.2.1 2022-03-13 18:53:54 +00:00
CalDescent
f0d9982ee4 Made arbitraryDataFileHashResponses final, and use .sort() rather than .stream() to avoid new instance creation. 2022-03-12 09:43:56 +00:00
CalDescent
c65de74d13 Revert "Synchronize arbitrary data list removals, as it seems that SynchronizedList and SynchronizedMap aren't overriding removeIf() with a thread-safe version."
This reverts commit e5f88fe2f4ce88a9851dedee53842b5639188f46.
2022-03-12 09:40:13 +00:00
CalDescent
df0a9701ba Improved logging in onNetworkGetArbitraryDataFileListMessage() 2022-03-11 16:51:19 +00:00
CalDescent
4ec7b1ff1e Removed time estimations that are no longer correct or relevant. 2022-03-11 16:50:34 +00:00
CalDescent
7d3a465386 Including the number of hashes (even if zero) is now required in GetArbitraryDataFileListMessage, to allow for additional fields. Enough peers should have updated by now for this to be ok. 2022-03-11 16:50:11 +00:00
CalDescent
30347900d9 Tidied up one last place that was accessing immutableConnectedPeers directly. This makes no difference, but helps with code consistency. 2022-03-11 15:28:54 +00:00
CalDescent
e5f88fe2f4 Synchronize arbitrary data list removals, as it seems that SynchronizedList and SynchronizedMap aren't overriding removeIf() with a thread-safe version. 2022-03-11 15:22:34 +00:00
CalDescent
0d0ccfd0ac Small refactor for code simplicity. 2022-03-11 15:11:07 +00:00
CalDescent
9013d11d24 Report as 100% synced if the latest block is within the last 30 mins
This should hopefully reduce confusion due to APIs reporting 99% synced even though up to date. The systray should never show this since it already treats blocks in the last 30 mins as synced.
2022-03-11 14:53:10 +00:00
CalDescent
fc5672a161 Use a more tolerant latest block timestamp in the isUpToDate() calls below to reduce misleading systray statuses.
Any block in the last 30 minutes is considered "up to date" for the purposes of displaying statuses.
2022-03-11 14:49:02 +00:00
CalDescent
221c3629e4 Don't refetch the file list if the fileListCache is empty, since an empty list now means that there are likely to be no files available on disk. 2022-03-11 13:08:37 +00:00
CalDescent
76fc56f1c9 Fetch the file list in getFilenameForHeight() if needed. 2022-03-11 13:07:16 +00:00
CalDescent
8e59aa2885 Peer getter methods renamed to include "immutable", for consistency with underlying lists and also to make it clearer to the callers. 2022-03-11 13:00:47 +00:00
CalDescent
0738dbd613 Avoid direct access to this.connectedPeers, as we need to use the immutable copy. 2022-03-11 12:58:11 +00:00
CalDescent
196ecffaf3 Skip calls to this.logger.trace() in ExecuteProduceConsume.run() if trace logging isn't enabled.
This could very slightly reduce load due to skipping the internal filtering inside log4j. Given that this method is causing major problems with CPU at times, I'm trying to make it as optimized as possible.
2022-03-11 11:59:18 +00:00
CalDescent
a0fedbd4b0 Implemented suggestions from catbref to avoid potential thread safety issue in peer arrays. 2022-03-11 11:27:13 +00:00
CalDescent
7c47e22000 Set fileListCache to null when invalidating. 2022-03-11 11:01:29 +00:00
CalDescent
6aad6a1618 fileListCache is now an immutable Map, which is thread safe. Thanks to catbref for this idea. 2022-03-11 10:59:07 +00:00
CalDescent
b764172500 Revert "Hopeful fix for ConcurrentModificationException in BlockArchiveReader.getFilenameForHeight()"
This reverts commit a12ae8ad241dc70828ba2132dc8ccaaea7f990a5.
2022-03-11 10:55:22 +00:00
CalDescent
c185d79672 Loop through all available direct peer connections and try each one in turn.
Also added some extra conditionals to avoid repeated attempts with the same port.
2022-03-09 20:55:27 +00:00
CalDescent
76b8ba91dd Only add an entry to directConnectionInfo if one with this peer-signature combination doesn't already exist. 2022-03-09 20:50:03 +00:00
CalDescent
0418c831e6 Direct connections with peers now prefer those with the highest number of chunks for a resource. Once a connection has been attempted with a peer, remove it from the list so that it isn't attempted again in the same round. 2022-03-09 20:15:26 +00:00
CalDescent
4078f94caa Modified GetArbitraryDataFileListMessage to allow requesting peer's address to be optionally included.
This can ultimately be used to notify the serving peer to expect a direct connection from the requesting peer (to allow it to temporarily bypass maxConnections for long enough for the files to be retrieved). Or it could even possibly be used to trigger a reverse connection (from the serving peer to the requesting peer).
2022-03-09 19:58:02 +00:00
CalDescent
a12ae8ad24 Hopeful fix for ConcurrentModificationException in BlockArchiveReader.getFilenameForHeight() 2022-03-09 19:46:50 +00:00
CalDescent
498ca29aab Wait until a successful connection with a peer before tracking the direct request. 2022-03-08 23:07:08 +00:00
CalDescent
ba70e457b6 Default chunk size reduced from 1MB to 0.5MB 2022-03-08 22:44:43 +00:00
CalDescent
d62808fe1d Don't attempt to create the data directory every time an ArbitraryDataFile instance is instantiated. This was using excessive amounts of CPU and disk I/O. 2022-03-08 22:42:07 +00:00
CalDescent
6c14b79dfb Removed bootstrap host that is no longer functional. 2022-03-08 22:30:01 +00:00
CalDescent
631a253bcc Added support for dark theme in loading screen. 2022-03-08 22:29:37 +00:00
CalDescent
4cb63100d3 Drop the ArbitraryPeers table as it's no longer needed 2022-03-06 13:01:09 +00:00
CalDescent
42fcee0cfd Removed all code that interfaced with the ArbitraryPeers table 2022-03-06 13:00:11 +00:00
CalDescent
829a2e937b Removed all arbitrary signature broadcasts 2022-03-06 12:58:01 +00:00
CalDescent
5d7e5e8e59 Dropped support of ARBITRARY_SIGNATURES message handling, as this feature has been superseded by the peerAddress in file list requests. 2022-03-06 12:46:06 +00:00
CalDescent
6f0a0ef324 Small refactor 2022-03-06 12:42:19 +00:00
CalDescent
f7fe91abeb sendOurOnlineAccountsInfo() moved to its own thread, in preparation for mempow 2022-03-06 12:41:54 +00:00
CalDescent
7252e8d160 Deleted presence tests, as they are no longer relevant, and aren't easily adaptable to the new approach. 2022-03-06 12:03:18 +00:00
CalDescent
2630c35f8c Chunk validation now uses MAX_CHUNK_SIZE rather than CHUNK_SIZE, to allow for a smaller CHUNK_SIZE value to be optionally used, without failing the validation of existing resources. 2022-03-06 11:43:28 +00:00
CalDescent
49f466c073 Added missing break; 2022-03-06 11:21:55 +00:00
CalDescent
c198f785e6 Added significant CPU optimizations to ArbitraryDataManager
- Slow down loops that query the db
- Check for new metadata every 5 minutes instead of constantly
- Check for new data every 1 minute instead of constantly

This could be further improved in the future by having block.process() notify the ArbitraryDataManager that there is new data to process. This would avoid the need for the frequent checks/loops, and only a single complete sweep would be needed on node startup (as long as failures are then retried). But I will avoid this additional complexity for now.
2022-03-06 11:21:39 +00:00
CalDescent
5be093dafc Fix for "Synchronizing null%" systray bug introduced in 3.2.0 2022-03-06 11:00:53 +00:00
CalDescent
2c33d5256c Added code accidentally missed out of commit 1b036b7 2022-03-05 20:44:01 +00:00
CalDescent
4448e2b5df Handle case when metadata isn't returned. 2022-03-05 17:39:13 +00:00
CalDescent
146d234dec Additional defensiveness in ArbitraryDataFile.fromHash() to avoid similar future bugs. 2022-03-05 17:25:48 +00:00
CalDescent
18d5c924e6 Fixed bug cased by fetchAllMetadata() 2022-03-05 17:25:14 +00:00
CalDescent
b520838195 Increased default maxNetworkThreadPoolSize from 20 to 32
This will hopefully offset some of the additional network demands from arbitrary data requests.
2022-03-05 17:24:55 +00:00
CalDescent
1b036b763c Major CPU optimization to block minter
Load sorted list of reward share public keys into memory, so that the indexes can be obtained. This is around 100x faster than querying each index separately (and the savings will increase as more keys are added).

For 4150 reward share keys, it was taking around 5000ms to query individually, vs 50ms using this approach.

The main trade off is that these 4150 keys require around 130kB of additional memory when minting (and this will increase proportionally with more minters). However, this one query was often accounting for 50% of the entire core's CPU usage, so the additional memory usage seems insignificant by comparison.

To gain confidence, I ran both old and new approaches side by side, and confirmed that the indexes matched exactly.
2022-03-05 16:10:43 +00:00
CalDescent
8545a8bf0d Automatically fetch metadata for all resources that have it. 2022-03-05 13:00:49 +00:00
CalDescent
f0136a5018 Include the external port when responding ArbitraryDataFileListRequests 2022-03-05 13:00:17 +00:00
CalDescent
6697b3376b Direct peer connections now use the on-demand data retrieved from file list requests, rather than the stale and incomplete ArbitraryPeerData. 2022-03-05 12:59:13 +00:00
CalDescent
ea785f79b8 Removed unnecessary synchronization 2022-03-04 19:02:30 +00:00
CalDescent
0352a09de7 New online accounts are now verified on the OnlineAccountsManager thread rather than on network threads. This is an attempt to reduce the amount of blocked network threads due to signature verification, and is necessary for the upcoming mempow addition. 2022-03-04 17:58:06 +00:00
CalDescent
5b4f15ab2e Transaction importing code moved to TransactionImporter controller class
As with online accounts, no logic changes other than moving transaction queue processing from the controller thread to its own dedicated thread.
2022-03-04 16:47:21 +00:00
CalDescent
fd37c2b76b Moved all online accounts code to a new OnlineAccountsManager controller class
There are no logic changes here other than moving performOnlineAccountsTasks() onto its own thread, so that it's not subject to anything that might be slowing down the main controller thread.
2022-03-04 16:24:04 +00:00
CalDescent
924aa05681 Optimized peer lists
- Removed synchronization from connectedPeers, and replaced it with an unmodifiableList.
- Added additional immuatable caches: handshakedPeers and outboundHandshakedPeers

This should greatly reduce the amount of time spent waiting around for access to the connectedPeers array, since it is now immediately accessible without needing to obtain a lock. It also removes calls to stream() which were consuming large amounts of CPU to constantly filter the connected peers down to a list of handshaked peers.

Thanks to @catbref for these great suggestions.
2022-03-04 15:14:12 +00:00
CalDescent
84b42210f1 Use ArbitraryDataFileRequestThreads only - instead of reusing file list response threads. 2022-03-04 13:34:16 +00:00
CalDescent
941080c395 Rework of arbitraryDataFileHashResponses to use a list rather than a map (limited to 1000) items. Sort the list by routes with the lowest number of peer hops first, to try and prioritize those which are easiest and quickest to reach. 2022-03-04 13:33:17 +00:00
CalDescent
35d9a10cf4 Avoid logging if there are no remaining transaction signatures to validate. There was too much log spam, none of which was particularly useful. 2022-03-04 12:03:58 +00:00
CalDescent
7c181379b4 Added more granularity to logging, to differentiate between signature validation and general processing/importing, as well as showing counts of the transactions being processed in each round. 2022-03-04 11:12:23 +00:00
CalDescent
f9576d8afb Further optimizations to Controller.processIncomingTransactionsQueue()
- Signature validation is now able to run concurrently with synchronization, to reduce the chances of the queue building up, and to speed up the propagation of new transactions. There's no need to break out of the loop - or avoid looping in the first place - since signatures can be validated without holding the blockchain lock.
- A blockchain lock isn't even attempted if a sync request is pending.
2022-03-04 11:05:58 +00:00
CalDescent
6a8a113fa1
Merge pull request #74 from catbref/presence-txns-removal
PRESENCE transactions changed to always fail signature validation
2022-03-04 10:33:11 +00:00
CalDescent
ef59c34165 Added missing "break" which was causing additional unnecessary debug logging. Originally introduced due to a merge conflict with the metadata branch. 2022-03-04 10:28:44 +00:00
CalDescent
a19e1f06c0
Merge pull request #73 from catbref/incoming-txns-rework
Reworking of Controller.processIncomingTransactionsQueue()
2022-03-04 09:45:29 +00:00
catbref
a9371f0a90 In Controller.processIncomingTransactionsQueue(), don't bother with 2nd-phase of locking blockchain and importing if there are no valid signature transactions to actually import 2022-03-03 20:32:27 +00:00
catbref
a7a94e49e8 PRESENCE transactions changed to always fail signature validation 2022-03-03 20:25:58 +00:00
catbref
affd100298 Reworking of Controller.processIncomingTransactionsQueue()
Main changes are:
* Check transaction signature validity in initial round, without blockchain lock
* Convert List of incoming transactions to Map so we can record whether we have validated transaction signature before to save rechecking effort
* Add invalid signature transactions to invalidUnconfirmedTransactions map with INVALID_TRANSACTION_RECHECK_INTERVAL expiry (~60min)
* Other minor changes related to List->Map change and Java object synchronization
2022-03-03 20:21:04 +00:00
CalDescent
fd6ec301a4 Updated AdvancedInstaller project for v3.2.0 2022-03-03 20:02:30 +00:00
CalDescent
5666e6084b Bump version to 3.2.0 2022-03-02 20:04:49 +00:00
CalDescent
69309c437e Tightened up the content security policy for non HTML files. 2022-03-01 20:36:34 +00:00
CalDescent
e392e4d344 Allow eval(), setTimeout(), etc, to enable various QDN sites to function correctly. The existing sandboxing should be locking this down enough already. Limited to .html and .htm files only. 2022-03-01 20:35:56 +00:00
CalDescent
bd53856927 Disabled auto fetching of metadata. To be re-enabled at a later date. 2022-03-01 20:26:09 +00:00
CalDescent
cbd1018ecf Allow identical data to be published if the metadata differs. 2022-03-01 20:22:47 +00:00
CalDescent
46606152eb /arbitrary/metadata/* endpoint now returns ArbitraryResourceMetadata rather than a raw JSON string. 2022-03-01 20:22:20 +00:00
CalDescent
e6f93e0a08 Added categoryName to ArbitraryResourceMetadata, along with the existing category ID 2022-03-01 20:19:08 +00:00
CalDescent
8d81f1822f Merge branch 'master' into qdn-metadata
# Conflicts:
#	src/main/java/org/qortal/controller/Controller.java
#	src/main/java/org/qortal/network/message/Message.java
2022-02-28 20:10:39 +00:00
CalDescent
5903607363
Merge pull request #72 from catbref/presence-v2
Presence v2
2022-02-27 22:01:59 +00:00
catbref
590a8f52db Remove future work comment from Controller 2022-02-27 16:57:26 +00:00
catbref
ecac47d1bc Also notify TradePresenceWebsocket (using TradePresenceEvent) when bridging old PRESENCE txns 2022-02-27 16:56:17 +00:00
catbref
3b477ef637 Fix JAXB marshalling error (duplicate tradeAddress) in TradePresenceWebSocket. No need to send signature. Make sure publicKey is sent in Base58, not Base64. 2022-02-27 16:56:17 +00:00
catbref
e2ef5b2ef3 Missed change from last commit: incorrect logic in TradePresenceWebSocket 2022-02-27 16:56:17 +00:00
catbref
1d59feeb72 Created /websockets/crosschain/tradepresence to replace /websockets/presence 2022-02-27 16:55:30 +00:00
catbref
c53dd31765 Tidy up of trade presence timestamp generation & checking. Added tests. Renamed "online trades" to "trade presences" 2022-02-27 16:54:42 +00:00
catbref
4c02081992 Tidy up TradeBot presence logging. Decorate API endpoints /crosschain/tradeoffers and /crosschain/trade with presence expiry timestamps 2022-02-27 16:54:42 +00:00
catbref
cb57af3c53 Bugfixes to online trade sigs + bridging from PRESENCE transactions 2022-02-27 16:54:42 +00:00
catbref
01d810fc00 Initial effort at migrating PRESENCE transactions to dedicated network messages 2022-02-27 16:54:42 +00:00
CalDescent
8c2a9279ee Return metadata in various /arbitrary APIs if the "includemetadata" parameter is included.
This is very inefficient and will soon be replaced with dedicated ArbitraryResources / ArbitraryMetadata tables. But this is acceptable in the short term, especially if limit and offset are used.
2022-02-27 09:09:18 +00:00
CalDescent
0d65448f3d Request all metadata automatically. 2022-02-27 08:20:39 +00:00
CalDescent
9da2b3c11a Don't respond to file list requests with just the metadata file.
We have the separate metadata protocol for this now.
2022-02-27 07:28:11 +00:00
CalDescent
95400da977 Fixed typo in various tests (copy and paste error) 2022-02-26 22:10:55 +00:00
CalDescent
dc41dc4c69 Tags now use an array of strings, rather than a single string. 2022-02-26 22:09:07 +00:00
CalDescent
a5c11d4c23 Reduced "Ignoring hash list request" logs from DEBUG to TRACE 2022-02-26 16:10:44 +00:00
CalDescent
878394535e Improvements relating to fetching metadata
- Rate limiter is disabled when using the API
- fetchArbitraryMetadata() returns the actual metadata content rather than a boolean
- Exceptions are thrown on certain errors, rather than returning null
2022-02-26 16:10:26 +00:00
CalDescent
35dba27a55 Fixed issue due to not updating arbitraryMetadataRequests when receiving the metadata file. 2022-02-26 16:07:06 +00:00
CalDescent
f22ad13fa9 Merge branch 'master' into qdn-metadata
This involved a slight rewrite to remove the "includeMetadataOnly" boolean. Metadata is now always excluded, otherwise it complicates the caching too much.

# Conflicts:
#	src/main/java/org/qortal/api/resource/ArbitraryResource.java
#	src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
2022-02-26 14:39:20 +00:00
CalDescent
aa2e5cb87b Merge branch 'hosted-resources-search' 2022-02-26 14:05:52 +00:00
CalDescent
7740f3da7e Small formatting tweaks, for consistency with existing code. 2022-02-26 14:05:28 +00:00
CalDescent
badb576991 Fixed exception when identifier is null. Also handling null names as this may be a future scenario. 2022-02-26 14:04:35 +00:00
CalDescent
c65a63fc7e Fixed "query" parameter error in swagger documentation 2022-02-26 13:59:53 +00:00
CalDescent
0111747016 Added debug logging of new file list stats. 2022-02-25 13:30:07 +00:00
CalDescent
eac4b0d87b Maintain backwards support for pre-3.2.0 peers by only including new file list message params when sending to newer peers.
These params are optional and the process will function without them, just less efficiently.
2022-02-25 12:24:02 +00:00
CalDescent
3dadce4da4 Renamed a reference 2022-02-25 12:24:02 +00:00
CalDescent
1864468818 Prefer the route with the least number of hops when relaying. 2022-02-25 12:24:02 +00:00
CalDescent
1a59379162 Optionally include requestTime, requestHops, peerAddress, and isRelayPossible flag in ArbitraryDataFileListMessage 2022-02-25 12:24:02 +00:00
CalDescent
31d34c3946 Updated testnet documentation 2022-02-25 11:08:37 +00:00
CalDescent
3cc394f02d
Merge pull request #70 from catbref/synchronizer-newchaintipevent
Modify TradeBot to trigger when chain tip changes instead of with every new block
2022-02-24 20:04:26 +00:00
CalDescent
53c4fe9e80 Fixed another ElectrumX issue found in unit tests.
Peers that were thought to be missing output address data may actually have just been using a different key - "address" instead of "addresses". Now reading the addresses from both keys, which may remove the need for the previously added checks.
2022-02-24 20:01:56 +00:00
CalDescent
d5521068b0 Fixed issue in earlier commit, found in unit tests. 2022-02-24 19:45:37 +00:00
CalDescent
a63ef4010d Disabled expired transaction data deletion code for now, as this was often causing data to be incorrectly deleted.
This will need to be re-enabled at some point, but only after it's modified to be much less aggressive.
2022-02-24 19:05:29 +00:00
CalDescent
cec3e86eef
Merge pull request #71 from catbref/transferprivs-fix
Very slightly relax validity checks for TRANSFER_PRIVS…
2022-02-24 19:02:20 +00:00
catbref
8950bb7af9 Very slightly relax validity checks for TRANSFER_PRIVS to allow for skeletal account records, e.g. due to CHAT transactions, but account last reference still needs to be null. Example at block height 736196 / 7 2022-02-24 09:13:51 +00:00
catbref
9e6fe7ceb9 Modify TradeBot, some related websockets, to trigger when chain tip changes instead of with every new block 2022-02-24 09:06:21 +00:00
CalDescent
c333d18cd0 Merge branch 'segwit' 2022-02-23 20:07:07 +00:00
CalDescent
0271ef69c9 When submitting a new transaction, treat the chain as "synced" if the latest block is less than 30 minutes old. Increased from around 7.5 minutes. 2022-02-23 20:06:55 +00:00
CalDescent
2d493a4ea2 Added logging when no addresses are returned for a bitcoiny transaction output. 2022-02-23 09:29:16 +00:00
CalDescent
e339ab856f Skip over Electrum servers that don't return any output addresses. Hopeful fix for BTC transactions that report a zero value due to incomplete data being returned from certain ElectrumX peers. 2022-02-23 08:34:38 +00:00
proto
782904a971 improvement to the search on hosted resources
1) use the cached version instead of rescanning all the files
2) separating the loading (which include files scanning) and listing logic
2022-02-22 17:54:08 +01:00
proto
a3753c01bc Add search functionality to hosted resources 2022-02-22 15:50:46 +01:00
CalDescent
d5c3921846 Only show the red "synchronizing" systray icon if the latest block isn't recent.
This should fix issue where the icon unnecessarily jumps between synced and synchronizing.
2022-02-21 22:34:13 +00:00
CalDescent
a2c462b3da Add <meta charset="UTF-8"> tag to websites. Fixed issue rendering emojis and other special characters. 2022-02-21 22:28:59 +00:00
CalDescent
8673c7ef6e Fixed bug in GET /peers/summary API 2022-02-21 22:28:18 +00:00
CalDescent
8d7be7757f Fixed incorrectly named tag. 2022-02-21 22:27:44 +00:00
proto
6b83927048 Persist MintingAccounts.json on minting accounts add/remove
this fix the behavior of the node, After adding or removing a minting account, allowing it to persist it to the backup folder
2022-02-21 16:17:17 +01:00
proto
e07adbd60e online accounts api call, fix level zero accounts
Added online zero level accounts to the response of /addresses/online/levels api endpoints
2022-02-21 15:40:10 +01:00
CalDescent
7798b8dcdc Keep items in arbitraryDataFileHashResponses if they are currently being requested by another thread. This should help to locate the higher numbered chunks from larger resources. 2022-02-20 11:33:09 +00:00
CalDescent
146e7970bf Synchronize this.allKnownPeers and this.connectedPeers in Network.requestDataFromPeer(), to make the method thread-safe.
This could be further improved by taking an immutable copy, but I'll leave this until the same approach is applied to other Network methods.
2022-02-20 11:04:33 +00:00
CalDescent
f4f7cc58e3 Removed unused import. 2022-02-20 10:44:59 +00:00
CalDescent
21b4b494e7 Renamed method. 2022-02-20 10:44:38 +00:00
CalDescent
7307844bee If UPnP is disabled in settings, close the existing external listenPort if a UPnP rule exists. 2022-02-20 10:44:20 +00:00
CalDescent
5d419dd4ec Handle case where funds are sent to and from the same bitcoiny deterministic wallet. 2022-02-19 17:45:24 +00:00
CalDescent
6d0db7cc5e Catch UncheckedIOException in findAllHostedPaths() which was seen when a file was deleted by another thread. 2022-02-19 17:18:51 +00:00
CalDescent
8de606588c Attempt to open the listen port (default 12392) using UPnP, if the local network supports it. 2022-02-18 20:11:00 +00:00
CalDescent
5842b1272d Add WaifUPnP-1.1 jar to project.
For future reference, the command used was:

mvn install:install-file -Dfile=/Users/user/Downloads/waifupnp-1.1/WaifUPnP.jar -DgroupId=com.dosse -DartifactId=WaifUPnP -Dversion=1.1 -Dpackaging=jar -DlocalRepositoryPath=lib
2022-02-18 20:05:14 +00:00
CalDescent
35b0a85818 Increased WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ to 50 2022-02-18 18:42:06 +00:00
CalDescent
fcdd85af6c Try a lookahead size of 20 (instead of 3) when asking Bitcoinj for the balance. 2022-02-18 18:39:47 +00:00
CalDescent
5aac2dc9df ONLINE_ACCOUNTS_V2_PEER_VERSION set to 3.2.0 2022-02-18 18:34:00 +00:00
CalDescent
17a9b4e442
Merge pull request #66 from catbref/network-online-accounts-v2
New network messages ONLINE_ACCOUNTS_V2 and GET_ONLINE_ACCOUNTS_V2.
2022-02-18 18:28:52 +00:00
CalDescent
becb0b37e6
Merge branch 'master' into network-online-accounts-v2 2022-02-18 17:12:44 +00:00
CalDescent
67ca876567 Log if we're unable to process the received file. 2022-02-18 15:35:37 +00:00
CalDescent
464ce66fd5 Moved deletion retry code into ArbitraryDataFile 2022-02-18 15:09:50 +00:00
CalDescent
3e505481fe Default minPeerVersion also increased to 3.1.0 2022-02-18 14:50:46 +00:00
CalDescent
c90c3a183e Block peers below 3.1.0 2022-02-18 14:50:29 +00:00
CalDescent
d1a7e734dc Updated AdvancedInstaller project for v3.1.1 2022-02-15 23:36:38 +00:00
CalDescent
6054982379 Bump version to 3.1.1 2022-02-15 20:02:21 +00:00
CalDescent
85b3278c8a Don't throttle the arbitrary data file request threads when there are items to process. 2022-02-15 19:39:26 +00:00
CalDescent
c90c287601 Increased ARBITRARY_REQUEST_TIMEOUT from 10 to 12 seconds, as some were coming back around 9-10 seconds later. 2022-02-15 19:38:30 +00:00
CalDescent
6ee395ed12 Stop bulk arbitrary signature broadcasts, as they were creating a lot of network traffic, and are in the process of being replaced with a better method. 2022-02-15 19:10:20 +00:00
CalDescent
6275ac2b81 Increased numberOfAdditionalBatchesToSearch from 5 to 7.
This is the equivalent of increasing the max address gap from 15 to 21. The electrum standalone wallet uses 20, so this should be the most we will ever need.
2022-02-14 22:58:37 +00:00
CalDescent
fd0a6ec71f Fix for invalid balance (and transaction amount) when there are no outputs relating to this wallet. 2022-02-14 22:53:30 +00:00
CalDescent
6c1c814aca Updated AdvancedInstaller project for v3.1.0 2022-02-14 20:15:37 +00:00
CalDescent
43791f00aa Wait 2 minutes on node startup before trying to fetch followed QDN data. 2022-02-14 19:33:58 +00:00
CalDescent
538ac30b4e Request only the missing hashes, not all of them. 2022-02-14 19:33:36 +00:00
CalDescent
58f11489db Bump version to 3.1.0 2022-02-13 20:32:19 +00:00
CalDescent
acddf36467 Handle missing includeMetadata parameter. 2022-02-13 19:27:12 +00:00
CalDescent
166d32032a Fixed message IDs. 2022-02-13 19:22:20 +00:00
CalDescent
e4238a62c9 Exclude metadata-only transactions in the data management page (but added an API parameter to allow them to optionally be included).
This ensures that the list will only show resources where there is at least 1 chunk.
2022-02-13 19:21:16 +00:00
CalDescent
ad9c466712 Fall back to UNCATEGORIZED if the parsed category doesn't match any available categories.
This allows for deletion of categories, as the resources will just move into UNCATEGORIZED until they are next updated.
2022-02-13 18:10:56 +00:00
CalDescent
a3d31bbaf1 Category updates based on feedback so far. 2022-02-13 17:56:47 +00:00
CalDescent
4821139501 Merge branch 'master' into qdn-metadata
# Conflicts:
#	src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
2022-02-13 15:50:12 +00:00
CalDescent
83213800b9 Use the timestamp from the registerNameTransactionData in unit tests, rather than the current time. 2022-02-13 15:05:28 +00:00
CalDescent
265ae19591 Fixed other failing tests due to increased REGISTER_NAME transaction fee. At some point we should determine the correct fee inside of generateBase(), but setting it explicitly adds confidence in testing for now. 2022-02-13 14:31:21 +00:00
CalDescent
c1598d20b5 Name registration fee increase timestamp set to Sunday, 20 February 2022 16:00:00 UTC 2022-02-13 13:47:00 +00:00
CalDescent
0712259057 Implemented REGISTER_NAME transaction fee increase from 0.001 to 5 QORT (average value based on community vote). 2022-02-13 13:45:48 +00:00
CalDescent
ea42a5617f Fixed ElectrumX log spam and errors 2022-02-13 10:58:45 +00:00
CalDescent
58a690e2c3 Route through new getAddressTransactions() wrapper. 2022-02-11 18:15:27 +00:00
CalDescent
3ae2f0086e Removed unusably slow electrum peer 2022-02-11 18:13:45 +00:00
CalDescent
19c83cc54d MAX_AVG_RESPONSE_TIME reduced to 500, as one peer regularly takes around 600ms to reply. 2022-02-11 18:12:34 +00:00
CalDescent
8ac298e07d Allow 3 retries for getTransaction() and getAddressTransactions() requests 2022-02-11 18:11:00 +00:00
CalDescent
9b43e4ea3d Time electrum requests, and move on to another server if one takes more than 1000ms on average to respond (measured over the last 5 requests). 2022-02-11 18:02:56 +00:00
CalDescent
dbacfb964b Increased TX_CACHE_SIZE from 200 to 1000, to speed up loading times on large wallets. 2022-02-11 16:55:29 +00:00
CalDescent
a664a6a790 Added more LTC Electrum peers from https://1209k.com/bitcoin-eye/ele.php?chain=ltc 2022-02-11 16:44:34 +00:00
CalDescent
ee1f072056 Improvement to last commit, so that caller class names are preserved. 2022-02-11 15:34:31 +00:00
CalDescent
a6aabaa7f0 Reduce build queue log spam by only logging high priority items (5 and above). 2022-02-11 15:28:41 +00:00
CalDescent
49b307db60 Treat a null priority as 0 2022-02-11 15:17:02 +00:00
CalDescent
f7341cd9ab Increased /arbitrary priority to 1 2022-02-11 15:13:53 +00:00
CalDescent
6932fb9935 Added "priority" property to build queue items.
/render APIs use priority 10, whereas /arbitrary use priority 0, to prevent thumbnail downloads from holding up website loading. The priorities can be adjusted later, with maybe some service types being given higher priority than others.
2022-02-11 15:08:12 +00:00
CalDescent
2343e739d1 Handle case where a file cannot be unzipped. 2022-02-11 14:35:46 +00:00
CalDescent
fc82f0b622 Use 5 builder threads, so that one slow resource (e.g. a thumbnail) doesn't hold up the other queued build items.
This can be replaced with a task-based approach longer term.
2022-02-11 13:58:45 +00:00
CalDescent
c0c50f2e18 Updated bootstrap hosts 2022-02-11 13:33:25 +00:00
CalDescent
9332d7207e Fixed bug in cache clearing logic, which was often preventing resource updates from being detected. 2022-02-10 09:22:54 +00:00
CalDescent
a8c79b807b Discard any uncommitted changes as a result of the higher weight chain detection 2022-02-10 08:16:30 +00:00
CalDescent
2637311ef5 Prevent potential ConcurrentModificationException in the build queue 2022-02-09 20:20:30 +00:00
CalDescent
06b5b8f793 Reduced time between processing build tasks, to prevent builds with invalid criteria from holding up legitimate builds too much. 2022-02-09 20:17:56 +00:00
CalDescent
61f58173cb Revert "Removed transaction caching. Can be reintroduced later."
This reverts commit 9804eccbf0aa1b5051afdf2e18d7b0554f931a71.
2022-02-09 19:46:20 +00:00
CalDescent
b7b66f6cba Revert "Removed getWalletTransactions() synchronization. Again, can be re-added later."
This reverts commit 70c864bc2fda7f84a2015ef93ee0f451ee07d305.
2022-02-09 19:46:16 +00:00
CalDescent
dda2316884 Revert "Try a lookahead size of 20 (instead of 3) when asking Bitcoinj for the balance."
This reverts commit d7658ee9f9bc8d7edef9e9189af1d0a129286a67.
2022-02-09 19:46:10 +00:00
CalDescent
b782679d1f Revert "Revert "Calculate wallet balances from the transactions (ElectrumX) rather than using bitcoinj.""
This reverts commit 214f49e356ae0a28289736b731a1d0582dc6fb28.
2022-02-09 19:46:06 +00:00
CalDescent
b0f19f8f70 Merge branch 'block-minter-updates'
# Conflicts:
#	src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java
2022-02-09 19:42:39 +00:00
CalDescent
de5f31ac58 Don't process file hashes if we're stopping 2022-02-09 19:40:20 +00:00
CalDescent
214f49e356 Revert "Calculate wallet balances from the transactions (ElectrumX) rather than using bitcoinj."
This reverts commit 892612c084ac34b0904d6deb72d5750d55e5abb7.

# Conflicts:
#	src/main/java/org/qortal/crosschain/Bitcoiny.java
2022-02-08 18:29:32 +00:00
CalDescent
d7658ee9f9 Try a lookahead size of 20 (instead of 3) when asking Bitcoinj for the balance. 2022-02-08 18:27:44 +00:00
CalDescent
70c864bc2f Removed getWalletTransactions() synchronization. Again, can be re-added later. 2022-02-08 18:27:08 +00:00
CalDescent
9804eccbf0 Removed transaction caching. Can be reintroduced later. 2022-02-08 18:26:15 +00:00
CalDescent
d1f24d45da Added defensiveness in convertToSimpleTransaction() 2022-02-08 18:24:42 +00:00
CalDescent
9630625449 Rework of processIncomingTransactionsQueue() so that it no longer holds the lock while processing.
This should fix an issue where network threads could be blocked when new transactions arrived, due to waiting for the incomingTransactions lock to free up.
2022-02-08 09:18:14 +00:00
CalDescent
b72153f62b Renamed main thread from "Controller" to "Qortal" 2022-02-08 09:02:20 +00:00
CalDescent
0a88a0c95e Perform the base58 decoding outside of the arbitraryDataFileHashResponses lock, to reduce the amount of waiting around by other threads. 2022-02-08 08:45:58 +00:00
CalDescent
ab4ba9bb17 Don't re-fetch unconfirmed transactions that are already in the queue 2022-02-08 08:36:45 +00:00
CalDescent
a49218a840 Optimized ArbitraryDataFileRequestThread - only start a database transaction when there's something to process. 2022-02-07 22:06:45 +00:00
CalDescent
b6d633ab24 Break out of incoming transactions processing loop if we need to sync. 2022-02-07 22:05:13 +00:00
CalDescent
133943cd4e Reduce log spam 2022-02-07 22:03:41 +00:00
CalDescent
f8ffb1a179 Updated thread names 2022-02-07 22:03:26 +00:00
CalDescent
41c4e0c83e Merge branch 'master' into block-minter-updates 2022-02-06 18:40:32 +00:00
CalDescent
99f6bb5ac6 Reorganized some controller methods. 2022-02-06 18:32:43 +00:00
CalDescent
3e0306f646 Increased minPeerConnectionTime and maxPeerConnectionTime to reduce the chances of forced connections during relays.
An alternate option would be to avoid force disconnecting while relays are in progress, but some nodes could have active relays 100% of the time and therefore would never recycle their peers. So it is simpler to just increase the average peer connection time for everyone.
2022-02-06 17:29:00 +00:00
CalDescent
84e4f9a1c1 Rework of arbitraryRelayMap to keep track of multiple responses.
Previously, only one peer's response for a hash would be remembered, even if multiple others reported back too. This would cause useful mapping to be lost.
2022-02-06 17:20:01 +00:00
CalDescent
cd5ce6dd5e Don't remove from the relay map after a file is requested, as it may be needed by other peers.
It will be cleaned up automatically after 60 seconds, so it is best to keep the data intact until then.
2022-02-06 16:04:54 +00:00
CalDescent
9ec4e24ef6 Slightly optimized logic in fetchArbitraryDataFiles() 2022-02-06 15:45:40 +00:00
CalDescent
fa447ccded Builder thread updates. 2022-02-06 15:37:21 +00:00
CalDescent
ef838627c4 Stop asking for hashes from a peer if one fails.
This fixes the request looping that occurs on when a peer is unable to serve files.
2022-02-06 15:37:08 +00:00
CalDescent
b8aaf14cdc Introduced ArbitraryDataFileRequestThread to allow for multiple concurrent file requests.
This is likely a short term solution (to allow existing code to be repurposed) until replaced with a task-based approach, as this will allow for a much greater number of threads.
2022-02-06 15:34:06 +00:00
CalDescent
2740543abf Added "async" and "attempts" parameters to GET /arbitrary/{service}/{name}* endpoints.
async = fail immediately with 404 if missing, and request in the background
attempts = the number of times to request the data (synchronous mode only for now)
2022-02-06 13:04:58 +00:00
CalDescent
3c526db52e Fixed bug in build manager which would prevent future builds until the core was restarted. 2022-02-06 13:03:01 +00:00
CalDescent
cfe0414d96 Small rework of invalidUnconfirmedTransactions to specify the expiry time instead of the time added.
This allows TIMESTAMP_TOO_OLD transactions to be tracked for a shorter time (10 minutes) than the other invalid transactions (60 minutes). Should reduce network traffic and db load around the time that transactions are expiring, as there is a lag before they are noticed and removed from each node. Due to the variance, it could cause other peers to request them again after deleting. They are now ignored for 10 minutes to avoid request spam.
2022-02-06 12:35:41 +00:00
CalDescent
08e06ba11a Fixed bugs preventing invalidUnconfirmedTransactions from working as intended. 2022-02-06 12:09:44 +00:00
CalDescent
8c03164ea5 Don't add expired transactions to invalidUnconfirmedTransactions, as there is no need to keep track of these. 2022-02-06 11:55:07 +00:00
CalDescent
0fe2f226bc Added invalidUnconfirmedTransactions map
An incoming invalid unconfirmed transaction will be added to this map if its timestamp is more than 30 minutes old. This should allow enough time and opportunities for it to be imported and included in a block (allowing for re-orgs which could switch its status from invalid to valid).

Once added, it will be removed after an hour to allow for another chance to be requested from any peers that still have it. If invalid again, it's added back to the map for another hour.

This fixes a 24 hour long loop, where invalid transactions are requested over and over from peers that have already imported them. It could be improved further by periodically removing invalid unconfirmed transactions from the database, but this will be a higher risk.

The results of this feature should be less network traffic, and less blockchain locks (which should ultimately increase the responsiveness of the synchronizer).
2022-02-06 11:23:28 +00:00
CalDescent
55b5702158 Invalidate last low weight block signature whenever the previous block data changes. 2022-02-05 17:54:36 +00:00
CalDescent
a4cbbb3868 Moved block minter sleep to later in the process, otherwise it can remain there for longer than expected. 2022-02-05 17:54:36 +00:00
CalDescent
816b01c1fc Fixed issue in rebase 2022-02-05 17:54:36 +00:00
CalDescent
483e7549f8 Revert "Moved log from INFO to DEBUG, as now the synchronizer is on its own thread it can occur more often than before."
This reverts commit e2e87766faa0676a6bd334572743f2951918a379.
2022-02-05 17:54:35 +00:00
CalDescent
60d71863dc Allow 3 seconds for the synchronizer to obtain a blockchain lock, to reduce missed attempts. 2022-02-05 17:54:35 +00:00
CalDescent
170244e679 More work on higher weight chain detection in the block minter.
Added 30 second timeout, so that any errors should self correct.
2022-02-05 17:54:35 +00:00
CalDescent
472e1da792 Added debug level logging in higherWeightChainExists() for better visibility. 2022-02-05 17:54:35 +00:00
CalDescent
cbf03d58c8 Made synchronizer method public as it is now also used by the block minter. 2022-02-05 17:54:35 +00:00
CalDescent
ba41d84af9 Initial attempt to avoid an unnecessary block submission if one of our peers already has a higher weight chain. 2022-02-05 17:54:35 +00:00
CalDescent
98831a9449 Break out of the various loops in the cleanup manager if the thread is stopping. 2022-02-05 17:53:49 +00:00
CalDescent
9692539a3f Don't include fee in balance calculation (it looks like it could be double counting at the moment). 2022-02-05 17:24:33 +00:00
CalDescent
76df332b57 Check for null IP address before notifying of an external IP update. 2022-02-05 11:51:54 +00:00
CalDescent
c6405340bc minAccountLevelForBlockSubmissions reduced from 6 to 5 2022-02-05 11:33:28 +00:00
CalDescent
775e3c065e Invalidate ElectrumX transactions cache when switching accounts. 2022-02-05 10:23:25 +00:00
CalDescent
8937b3ec86 Don't allow duplicate transaction in the incoming transactions queue.
This should reduce database load slightly, as it won't have to check the same transaction multiple times in each batch.
2022-02-05 10:19:26 +00:00
CalDescent
3fbb86fded Added indexes, to make looking up name transactions by name around 5x faster. 2022-02-04 16:27:31 +00:00
CalDescent
0cf2f7f254 Missing import from last commit 2022-02-04 16:18:00 +00:00
CalDescent
9e571b87e8 Yet another rewrite of fetchAllTransactionsInvolvingName() - this time over 1000x faster since it doesn't involve joining the Transactions table. 2022-02-04 16:17:31 +00:00
CalDescent
23bafb6233 Removed unused methods 2022-02-04 14:00:44 +00:00
CalDescent
6dec65c5d9 Rewrite of fetchAllTransactionsInvolvingName() to avoid having to load all name transactions into memory. 2022-02-04 13:58:05 +00:00
CalDescent
4e59eb8958 Added unit test to simulate the false association between previous a UPDATE_NAME transaction, and the emoji name with a blank reducedName. 2022-02-04 12:51:22 +00:00
CalDescent
756d5e685a Added naming tests for blank new names 2022-02-04 12:50:05 +00:00
CalDescent
f52530b848 More thorough approach to fetchAllTransactionsInvolvingName(), to fix an issue found in unit testing. 2022-02-04 11:58:43 +00:00
CalDescent
c2bf37b878 Added transactionV5Timestamp featureTrigger to unit tests 2022-02-04 11:37:41 +00:00
CalDescent
98a2dd04b8 Fixed bug caused by improper handling of UPDATE_NAME transactions, similar to commit d16663f. 2022-02-04 10:28:26 +00:00
CalDescent
694ea689c8 Synchronize the loop and break out of it before fetching arbitrary data files. Hopeful fix for ConcurrentModificationException, and maybe a potential deadlock. 2022-02-03 21:14:41 +00:00
CalDescent
618aaaf243 Removed logs used for debugging only 2022-02-03 21:06:36 +00:00
CalDescent
9224ffbf73 Cache transaction list for 2 minutes, and synchronize, to prevent the balance and transactions APIs both requesting at once.
This ensures that only a single round of requests (per coin) is used for the wallettransactions and balance APIs. It also speeds up loading on subsequent requests. The 2 minute cache isn't much longer than the foreign block times, so shouldn't cause values to be too out of date.
2022-02-03 21:04:49 +00:00
CalDescent
892612c084 Calculate wallet balances from the transactions (ElectrumX) rather than using bitcoinj.
Hopeful fix for incorrect balances in wallets with large numbers of transactions. At the very least, this gives us control of the code that calculates the balance.
2022-02-03 20:22:25 +00:00
CalDescent
077165b807 Modified fetchArbitraryDataFileList() to support requesting only the missing hashes, but it is not yet used.
Once GetArbitraryDataFileListMessage is rolled out to the network we can uncomment this code. Needs testnet testing prior to that.
2022-02-03 20:01:44 +00:00
CalDescent
7994fc6407 Rework of onNetworkGetArbitraryDataFileListMessage() to support custom hashes to be optionally supplied.
Also simplified the existing logic, to make the code more readable.
2022-02-03 19:58:50 +00:00
CalDescent
d98df3e47d Added support for file hashes to optionally be included in GetArbitraryDataFileListMessage.
Needs testing on testnet, as the majority of nodes need to update before the hashes can be used.
2022-02-03 19:55:22 +00:00
CalDescent
1064b1a08b Removed duplicate metadata file checks. 2022-02-03 19:53:29 +00:00
CalDescent
0b7a7ed0f1 Added some more debug logging relating to file responses. 2022-02-03 19:52:28 +00:00
CalDescent
114b1aac76 Added arbitrary data file manager thread, which will ensure that all file list responses are tried until we receive the files.
Previously we would only try the first response and then discard the others due to being duplicates. They are now added to a queue and retried by the dedicated thread (up to the 60 second timeout).
2022-02-03 19:51:02 +00:00
CalDescent
6d06953a0e Increased RELAY_REQUEST_MAX_HOPS from 3 to 4, in an attempt to reach more peers. 2022-02-02 22:31:40 +00:00
CalDescent
0430fc8a47 Give up immediately after a synchronization if we are shutting down the synchronizer 2022-02-02 09:18:46 +00:00
CalDescent
7338f5f985 Attempt to acquire a blockchain lock (for up to 5 seconds) before shutting down the repository.
This should fix conflicts caused by the synchronizer and controller now being on separate threads. It may also reduce the chances of the database corrupting on shutdown, but this remains to be seen.
2022-02-02 09:17:24 +00:00
CalDescent
640bcdd504 Shutdown/interrupt the synchronizer as early as possible 2022-02-02 09:13:49 +00:00
CalDescent
c9d5d996e5 Increased accuracy of the block weights in the synchronizer logs, as the extra precision is now needed for debugging. 2022-02-01 22:06:01 +00:00
CalDescent
710befec0c Increased ARBITRARY_RELAY_TIMEOUT from 30 to 60 seconds, so that relay peers remember their mappings for longer. 2022-02-01 22:04:53 +00:00
CalDescent
8ccb158241 Reduce log spam when in DEBUG mode. 2022-02-01 22:04:12 +00:00
CalDescent
97199d9b91 Display the local and total chunk counts on the loading screen. 2022-02-01 22:03:52 +00:00
CalDescent
5a8b895475 Fixed bug in loading screen, which prevented the DOWNLOADING status from showing. 2022-02-01 22:03:17 +00:00
CalDescent
6c9600cda0 Added API key header to GET /arbitrary/resource/status/* endpoints 2022-02-01 22:02:41 +00:00
CalDescent
82fa6a4fd8 Include "localChunkCount" and "totalChunkCount" in the GET /arbitrary/resource/status/* API responses.
These values are left out of other API endpoints where multiple resources are returned, because calculating the chunk counts is too time consuming.
2022-02-01 22:02:14 +00:00
CalDescent
45f2d7ab70 Revert "We (currently) can't filter unconfirmed transactions by address, because only the public key is stored in the database until it is confirmed (at which point there is an entry in the TransactionParticipants table which contains the address). Given that this isn't a simple problem to solve, for now it makes sense to reject this combination if requested via the /transactions/search API."
This reverts commit 7aed0354f1e2926baa4257d4fbb2a04d14e195c2.
2022-02-01 21:55:41 +00:00
CalDescent
33731b969a Direct peer connections now send a file list request to the peer, rather than individually requesting every chunk for a transaction. 2022-02-01 09:04:31 +00:00
CalDescent
40a8cdc71f Improved logging when fetching data files 2022-01-31 22:35:12 +00:00
CalDescent
cbe83987d8 Fixed occasional ConcurrentModificationException in the block archive reader. 2022-01-31 21:39:49 +00:00
CalDescent
01e4bf3a77 Try to speed up the shutdown process of the cleanup manager. 2022-01-31 21:39:20 +00:00
CalDescent
b198a8ea07 Fixed issue in ArbitraryDataFile.chunkExists() due to it not checking for the metadata file. 2022-01-31 21:38:51 +00:00
CalDescent
e2e87766fa Moved log from INFO to DEBUG, as now the synchronizer is on its own thread it can occur more often than before. 2022-01-30 20:15:43 +00:00
CalDescent
f005a0975d Shut down the synchronizer as soon as the controller is stopping, if we are able to. 2022-01-30 20:14:11 +00:00
CalDescent
5700369935 Prioritize syncing over transaction importing. 2022-01-30 20:09:35 +00:00
CalDescent
8a1fb6fe4e If a lock can't be obtained when synchronizing, automatically request the sync again. 2022-01-30 20:09:09 +00:00
CalDescent
5b788dad2f Relocated all synchronizer code from the controller to the synchronizer, and also moved synchonization onto its own thread. 2022-01-30 20:01:22 +00:00
CalDescent
fa2bd40d5f Reduce log spam 2022-01-30 19:51:49 +00:00
CalDescent
074bfadb28 Don't bother locking if there are no new transactions to process 2022-01-30 19:50:09 +00:00
CalDescent
bd60c793be Incoming transactions are now added to a queue, and then processed soon after.
This solves a problem where incoming transactions could rarely obtain a blockchain lock (due to multiple transactions arriving at once) and therefore most messages were thrown away. It was also causing constant blockchain locks to be acquired, which would often prevent the synchronizer from running.
2022-01-30 19:03:31 +00:00
CalDescent
90f3d2568a Log whenever the synchronizer can't obtain a blockchain lock, so that blockchain lock issues are more easily noticed. 2022-01-30 19:00:28 +00:00
CalDescent
c73cdefe6f transactionV5Timestamp moved to blockchain.json 2022-01-30 13:47:20 +00:00
CalDescent
c5093168b1 minAccountLevelToMint value moved to blockchain.json 2022-01-29 22:59:04 +00:00
CalDescent
a35e309a2f minAccountLevelForBlockSubmissions moved to blockchain.json 2022-01-29 22:57:22 +00:00
CalDescent
d4ff7bbe4d Fixed inaccurate README text that was accidentally merged from the data node repository. 2022-01-29 22:37:27 +00:00
CalDescent
d4d73fc5fc
Merge pull request #67 from otaku/docker
feat: add Dockerfile
2022-01-29 20:15:06 +00:00
CalDescent
bb35030112
Merge pull request #65 from QuickMythril/reduce-doge-fee
reduce DOGE fees
2022-01-29 20:14:04 +00:00
CalDescent
7aed0354f1 We (currently) can't filter unconfirmed transactions by address, because only the public key is stored in the database until it is confirmed (at which point there is an entry in the TransactionParticipants table which contains the address). Given that this isn't a simple problem to solve, for now it makes sense to reject this combination if requested via the /transactions/search API. 2022-01-29 19:24:28 +00:00
CalDescent
c4f763960c Don't delete a resource's cache if a build is in progress.
Hopeful fix for "Unable to delete cache for resource: Unable to delete directory" error, and possibly some other file conflicts.
2022-01-29 19:18:06 +00:00
CalDescent
c5182a4589 Increased MAX_ACCOUNT_COUNT in GetOnlineAccountsMessage from 1000 to 5000 2022-01-29 12:39:43 +00:00
CalDescent
fc1a376fbd Added POST /transaction/fee API endpoint, to return the recommended fee for the supplied transaction data. 2022-01-29 10:38:15 +00:00
CalDescent
27387a134f Fixed typo 2022-01-28 19:58:32 +00:00
CalDescent
be7bb2df9e Added GET /transaction/unitfee API endpoint, to obtain the unit fee for a transaction type
Additional params:
- timestamp: to allow for hard forks. Default: the current time
- level: the account level, to allow for the future possibility of different fees per level. Not currently used.
2022-01-28 19:00:15 +00:00
CalDescent
72a291a54a Added initial support of different unit fees per transaction type.
Included a timestamp property, as this will be needed for each update (hard fork).
2022-01-28 18:50:03 +00:00
CalDescent
b1342d84fb Updated potentially misleading log message. 2022-01-28 11:02:37 +00:00
CalDescent
cdd57190ce Use getEffectiveMintingLevel() rather than getLevel() 2022-01-28 10:16:42 +00:00
CalDescent
d200a098cd Updated AdvancedInstaller project for v3.0.4 2022-01-27 23:18:34 +00:00
CalDescent
a0ed3f53a4 Merge branch 'master' of github.com:Qortal/qortal 2022-01-27 22:45:40 +00:00
CalDescent
e5c12b18af Bump version to 3.0.4 2022-01-27 19:58:39 +00:00
CalDescent
7808a1553e Fixed case sensitive ordering issues in other aspects of QDN. 2022-01-27 18:46:59 +00:00
CalDescent
a0ba016171 Fixed case sensitive ordering issue in websites list. 2022-01-27 18:42:52 +00:00
CalDescent
344704b6bf MIN_LEVEL_FOR_BLOCK_SUBMISSION temporarily increased to 6.
This is to hopefully improve network stability whilst a more advanced solution is being worked on. It also allows us to collect some data on how well the network behaves when there are less block candidates. It should have no effect on minting rewards (other than any side effects as a result of improved network stability).
2022-01-27 18:15:00 +00:00
CalDescent
3303e41a39 Fixed unhandled case in GET /arbitrary/{service}/{name}* endpoints 2022-01-27 18:12:21 +00:00
CalDescent
4e71ae0e59 Allow QDN data to be served without authentication by setting "qdnAuthBypassEnabled":true
This allows the GET /arbitrary/{service}/{name} and GET /{service}/{name}/{identifier} endpoints to operate without any authentication. Useful for those who are running public QDN nodes and need to serve data over http(s).
2022-01-27 18:10:24 +00:00
CalDescent
9daf7a6668 Synchronize lists, to prevent an occasional ConcurrentModificationException 2022-01-26 22:40:34 +00:00
Alexander Do
a2b2b63932 feat: add Dockerfile 2022-01-23 16:48:34 -08:00
CalDescent
af06774ba6 Clear the cache when deleting data, so that it disappears from the data management screen. 2022-01-23 22:53:35 +00:00
catbref
244d4f78e2 New network messages ONLINE_ACCOUNTS_V2 and GET_ONLINE_ACCOUNTS_V2.
Increased GetOnlineAccountsMessage.MAX_ACCOUNT_COUNT from 1000 to 5000.

The V2 versions are more efficiently encoded and also cache the payload bytes
which reduces CPU when sending to multiple peers.

Serialization / deserialization unit tests included.

Tentative V2 message activation set at core version 3.1.2
see Controller.ONLINE_ACCOUNTS_V2_PEER_VERSION
2022-01-23 16:24:10 +00:00
CalDescent
311fe98f44 Bump version to 3.0.3 2022-01-23 13:19:49 +00:00
CalDescent
6f7c8d96b9 Removed ApiService instance creation in ApplyUpdate as it wasn't really needed, and probably not sensible to instantiate it here. 2022-01-23 12:57:28 +00:00
CalDescent
ff6ec83b1c Removed localAuthBypassEnabled override in unit tests.
Hopefully this will allow us to proactively catch any missing API keys in the future.
2022-01-23 12:48:37 +00:00
CalDescent
ea10eec926 Fixed bug in auto update process - use the API key when stopping the node.
Luckily this code is included in the new JAR, not the old one, so we should be able to regain auto update ability by issuing a new update.
2022-01-23 12:38:19 +00:00
CalDescent
be561a1609 Add default "dataPath" to Windows installer builds, so that QDN data is located in AppData 2022-01-22 22:27:14 +00:00
CalDescent
6f724f648d Fixed testDirectoryDigest() which has been failing for a couple of versions (due to gitignore removing the cache file) 2022-01-22 20:48:58 +00:00
CalDescent
048776e090 Ignore failing test due to recent API update, which makes the test incompatible. To be fixed later. 2022-01-22 20:43:28 +00:00
CalDescent
dedf65bd4b Added initial protocol methods for metadata requests and forwarding. Not tested yet. 2022-01-22 20:24:37 +00:00
CalDescent
a7c02733ec Updated approve-auto-update.sh to use new service format 2022-01-22 19:40:13 +00:00
CalDescent
59346db427 Bump version to 3.0.2 2022-01-22 18:45:32 +00:00
CalDescent
25efee55b8 Added networking optimization, to avoid wasted processing on every read.
Thanks to @catbref for finding this.
2022-01-22 18:43:32 +00:00
CalDescent
a79ed02ccf Added initial (unfinished) category list, as well as the GET /arbitrary/categories API, and converted the category field from a string to an enum 2022-01-22 12:11:16 +00:00
CalDescent
79f87babdf Limit the metadata string lengths 2022-01-21 22:52:31 +00:00
CalDescent
f296d5138b Allow metadata to optionally be included with any arbitrary resource. 2022-01-21 21:14:28 +00:00
CalDescent
b30445c5f8 Increase MAX_ACCOUNT_COUNT to 5000 (around 0.5MB of data) to see if it helps with minting efficiency. 2022-01-21 16:11:21 +00:00
CalDescent
d105613e51 Improvements to some /blocks API endpoints 2022-01-21 16:07:18 +00:00
CalDescent
ef43e78d54 Reduced log spam 2022-01-21 15:20:16 +00:00
CalDescent
6f61fbb127 Reduced strictness of rate limiter, to allow two additional file list requests (15 and 30 seconds after initial attempt)
This should help catch any remaining chunks that were unable to be fetched in the first attempt due to network failures, bad peers, etc.
2022-01-21 15:20:04 +00:00
CalDescent
9f9b7cab99 Include "size" in /arbitrary/resource APIs
Note that this is not always accurate - it relates to the largest transaction size for this name, not necessarily the latest or the combined size of multiple transactions. This can be made accurate as soon as we have a "Resources" table to store this info. Trying to do it before then will be too inefficient in terms of queries.
2022-01-21 14:42:37 +00:00
CalDescent
f129e16878 Added retry mechanism to relay file deletions, just in case it fails on the first try.
A longer term solution to this hypothetical problem is to store relays in RAM or a temp folder only. Or maybe add an indicator file to instruct the cleanup manager to delete it. But this will require more development. 10 deletion attempts (each 1 second apart) should be enough for now.
2022-01-21 14:32:15 +00:00
CalDescent
8a42dce763 Use the first responding peer in the relay map.
This encourages shorter relays, since longer ones will take more time to respond, and also prevents a peer from intentionally taking a long time to respond so that it overwrites an existing entry.

Longer term we could consider keeping track of all respondents for each hash, if there are still issues with data retrieval. I suspect this won't be needed though, as the requesting peer has 16+ different peers connected, and therefore potentially 16 different mappings already.
2022-01-21 14:23:55 +00:00
CalDescent
6423d5e474 Optimized onNetworkGetArbitraryDataFileListMessage() to remove duplicate calls to Files.exists() 2022-01-21 14:12:59 +00:00
CalDescent
6e91157dcf Improved cache clearing process and logging. 2022-01-21 13:32:59 +00:00
CalDescent
85c61c1bc1 Added GET /arbitrary/resources/search API
Example usage

List all websites with a name containing the word "crow" (using default identifier):
http://localhost:12391/arbitrary/resources/search?service=WEBSITE&query=crow&default=true&limit=20

List all resources with name or identifier containing the word "crow":
http://localhost:12391/arbitrary/resources/search?query=crow&default=false&limit=20
2022-01-21 10:04:18 +00:00
CalDescent
54af36fb85 Remove duplicates in GET /arbitrary/hosted/resources response 2022-01-20 22:34:41 +00:00
CalDescent
fcdcc939e6 Sort hosted data in reverse order (newest first) 2022-01-20 22:29:19 +00:00
CalDescent
13450d5afa Added limit/offset to GET /arbitrary/hosted API endpoints 2022-01-20 22:28:28 +00:00
CalDescent
5e1e653095 Removed unnecessary database lookups in GET /hosted/resources API 2022-01-20 20:39:46 +00:00
CalDescent
e8fabcb449 Removed extra isDataLocal() check in GET /hosted/resources which was evading the cache. 2022-01-20 20:38:57 +00:00
CalDescent
a4ce41ed39 Updated qort and qdata to check for apikey.txt in $HOME/qortal, which is the most commonly installed location.
This allows the script to be run from any directory, as long as the core is installed at $HOME/qortal.
2022-01-20 20:31:16 +00:00
CalDescent
1b42062d57 Default minPeerVersion set to 3.0.1 2022-01-20 20:25:20 +00:00
CalDescent
c2a4b01a9c Allow local playback of media files 2022-01-20 20:25:06 +00:00
CalDescent
47e763b0cf Take a copy of the IP address history so it can be safely iterated. Without this, another thread could remove an element, resulting in an exception. 2022-01-20 20:24:51 +00:00
CalDescent
0278f6c9f2 Reduced more log spam 2022-01-20 20:24:26 +00:00
CalDescent
d96bc14516 Allow execution of inline scripts, at least for now. 2022-01-17 20:25:25 +00:00
CalDescent
318f433f22 Reduced log spam when checking for avatars. 2022-01-17 20:04:54 +00:00
CalDescent
cfc80cb9b0 Use a header instead of a meta tag for Content-Security-Policy, because we can't guarantee that we are parsing all HTML files.
Also use default-src instead of connect-src, as we want to block all external requests.
2022-01-17 20:04:35 +00:00
CalDescent
01c6149422 Restrict websites to same origin requests only, using a Content-Security-Policy meta tag. 2022-01-16 20:52:30 +00:00
CalDescent
6f80a6c08a Rework of file list requests and relays, allowing it to handle multiple chunk resources in a much more sensible way.
This could create a lot of additional relay traffic as a result, so needs lots of testing and possibly optimizing.
2022-01-16 20:39:37 +00:00
CalDescent
8fb2d38cd1 Revert "Revert log4j version for now. We need to put this back in the next update, once log4j2.properties files have transitioned to the new format."
This reverts commit 777bddd3d8bd7c56c447679f7c8b37d67a909bc6.
2022-01-15 20:35:44 +00:00
CalDescent
5018d27c25 "Not started" renamed to "Published" 2022-01-15 20:21:52 +00:00
CalDescent
1d77101253 Use AES/CBC/PKCS5Padding for encryption, and fall back to just AES for legacy resource support.
Should fix "ECB mode cannot use IV" error due to mode and padding not being stated.
2022-01-15 20:14:32 +00:00
CalDescent
1ddd468c1f Added API key support to qdata script
As with the qort script, it currently needs to be run from either the qortal directory or the tools directory in order to pick up the API key
2022-01-14 11:46:26 +00:00
CalDescent
f05cd9ea51 Added API key support to qort script
Currently needs to be run from either the qortal directory or the tools directory in order to pick up the API key
2022-01-14 11:45:40 +00:00
CalDescent
70c00a4150 Updated AdvancedInstaller project for v3.0.1 2022-01-13 22:36:31 +00:00
CalDescent
d296029e8e Bump version to 3.0.1 2022-01-13 20:18:32 +00:00
CalDescent
e257fd8628 Updated stop.sh script to use the /admin/stop API endpoint if an apikey.txt file is available.
This brings the behaviour closer to the old version so should hopefully reduce the amount of newly introduced issues. If an API key is unavailable, it will fall back to using `kill -15 $pid` (i.e. a SIGTERM).
2022-01-13 19:18:39 +00:00
CalDescent
119c1b43be Use default values for method and compression if not specified.
Should fix issue with v4 transactions where these aren't used. Matches with the NOT NULL DEFAULT 0 which automatically transitions existing v4 ARBITRARY transactions to use the same defaults.
2022-01-13 19:09:00 +00:00
CalDescent
1277ce38de Bump version to 3.0.0 2022-01-12 21:11:02 +00:00
CalDescent
6761b91400 QDN go-live timestamp set to Fri, 14 Jan 2022 16:00:00 UTC 2022-01-12 20:53:57 +00:00
CalDescent
2a6244a5c2 Handle multiple qortal processes in stop.sh 2022-01-12 20:31:21 +00:00
CalDescent
777bddd3d8 Revert log4j version for now. We need to put this back in the next update, once log4j2.properties files have transitioned to the new format. 2022-01-12 20:28:23 +00:00
CalDescent
e2b13791bb Don't reload the log4j2.properties file as this seems to be buggy in a lot of cases. 2022-01-12 20:26:41 +00:00
CalDescent
f44c21ce59 Disallow any kind of website/app/data rendering when localAuthBypassEnabled is enabled.
This allows node operators to return their authentication to the legacy rules (local requests allowed), without introducing javascript vulnerabilities. The websites, apps, etc are just prevented from loading, to avoid the risk of any API calls from javascript.
2022-01-12 19:32:52 +00:00
CalDescent
ade977e416 Don't broadcast any arbitrary signatures if the list is empty (i.e. the node isn't yet hosting anything) 2022-01-12 19:23:36 +00:00
CalDescent
f09a131bd6 Added requestHops to log entry. 2022-01-12 19:21:11 +00:00
CalDescent
4815587de1 Use V2 of string serialization methods in ArbitrarySignaturesMessage, as it is designed to allow null values. 2022-01-12 19:20:49 +00:00
CalDescent
e0ebfb9b53 Reduced log spam. 2022-01-12 19:20:14 +00:00
CalDescent
90836afd91 External IP address updates now require 10 consecutive readings. 2022-01-12 19:19:56 +00:00
CalDescent
4e1b0a25bb Fixed arbitrary peer tests, which used a local address (but we now treat them as invalid). 2022-01-11 20:06:14 +00:00
CalDescent
89c3236bf5 Updated bundled log4j2.properties files 2022-01-11 20:02:42 +00:00
CalDescent
7658bc2025 Added X-API-KEY header field to API documentation endpoints that require it. 2022-01-11 19:13:56 +00:00
CalDescent
7cf60c7c35 Updated stop.sh so that it no longer uses the core API. 2022-01-11 19:12:56 +00:00
CalDescent
ccde725d3b Check for an empty string as well as null in a couple of places, so that deserializeSizedString() can be safely used. 2022-01-10 19:42:21 +00:00
CalDescent
e3b45cac0a Use an alternative version of Serialization.serializeSizedString() and Serialization.deserializeSizedString() for the new ARBITRARY transaction additions.
The modifications made to these methods were causing issues with other transaction types that were expecting blank strings instead of null. To keep risk to a minimum, I have split into two different sets of functions until there is more time to unify them.
2022-01-10 19:41:02 +00:00
CalDescent
8f8a500dcd Fixed some issues left over from the qortaldata project 2022-01-10 19:26:25 +00:00
CalDescent
f9749cd82c Merge remote-tracking branch 'qortal-data/master' into qdn 2022-01-09 21:10:48 +00:00
CalDescent
051052fdd2 Removed authentication for GET /peers/summary endpoint 2022-01-09 21:09:20 +00:00
CalDescent
940304b4c2 Added GET /admin/apikey/test endpoint, so that we have a dedicated place to test if authentication works. 2022-01-09 20:08:45 +00:00
CalDescent
b4d2fae27f Fixed a couple of FOLLOWED_AND_VIEWED references that were missed 2022-01-09 16:26:52 +00:00
CalDescent
11e194292c Removed API key requirement from GET /admin/status and GET /admin/mintingaccounts 2022-01-09 16:26:23 +00:00
CalDescent
5ba6f6f53e FOLLOWED_AND_VIEWED renamed to FOLLOWED_OR_VIEWED, since it's technically an OR not an AND. 2022-01-09 13:25:49 +00:00
CalDescent
f58a16905f Removed unused setting. 2022-01-09 13:19:07 +00:00
CalDescent
33e82b336b Limit arbitrary signature requests to 3 hops, just in case a bug caused any kind of circular broadcasting. 2022-01-09 11:22:27 +00:00
CalDescent
0ced712974 Merge remote-tracking branch 'qortal-data/master' into qdn 2022-01-08 12:29:48 +00:00
CalDescent
db8e35cc13 Allow a new API key to be generated if the existing apikey.txt file has been deleted 2022-01-08 12:27:24 +00:00
CalDescent
b6db5aa2d3 Use "apikey.txt" instead of "apikey" as the filename to store the api key, to make it easier for users to open. 2022-01-08 10:22:14 +00:00
CalDescent
396dc5c9b0 Always log "Synchronizing with peer..." as it may help give more clarity to those with sync issues. 2022-01-08 10:12:54 +00:00
CalDescent
67e424a32a Added GET /arbitrary/relaymode API endpoint, which returns whether relay mode is enabled in the settings or not. 2022-01-07 14:38:05 +00:00
CalDescent
d8cbec41d2 Various logging improvements and fixes. 2022-01-07 14:08:11 +00:00
CalDescent
374f6b8d52 Added restrictions when relaying file list requests
1) Each relay request expires after 5 seconds, after which nodes will stop relaying it, preventing any kind of infinite loop. So it has to reach the destination peer within 5 seconds. This should be fine, because the original peer's request would timeout anyway, so there's nothing to be gained by continuing to relay it.

2) Each relay request stops being forwarded after 3 "hops" - i.e. once it has been relayed through 3 different peers, it will no longer be transmitted any further. If we assume that each node has 16 connections, that allows it to reach a theoretical maximum of 4096 peers in 3 hops. In practice it will be less, and may not reach everyone due to peer "islands". But it will automatically retry a few times on a timer, so should hopefully find what it needs eventually. Plus, it still has the ability to make a direct connection to anyone hosting the data, as long as they are port forwarded.
2022-01-07 14:01:57 +00:00
CalDescent
20ec4cbd14 ARBITRARY_REQUEST_TIMEOUT increased from 6 to 10 seconds
This is likely longer than needed, but it's best to allow extra for now and then optimize the timeouts once we've had some experience with real world data.
2022-01-07 12:50:34 +00:00
CalDescent
1c80835f49 Default relayModeEnabled to true.
Even though a final decision is yet to be made, it makes sense to test with this scenario to ensure that everything works correctly.
2022-01-07 12:31:16 +00:00
CalDescent
5e0af26c27 Keep track of successes or failures for a particular arbitrary peer / signature combination.
This can help to inform decisions on data retention (although there is no deletion yet).
2022-01-05 21:23:29 +00:00
CalDescent
b42674ac06 Small code reorganization to improve logic when adding arbitrary peer data 2022-01-05 19:26:06 +00:00
CalDescent
3394543705 Don't save arbitrary peer data if it's a local address 2022-01-05 19:22:24 +00:00
CalDescent
75c51aa61b If a direct connection can't be made to a peer over the original or default port, match the peer's host with any entries in knownPeers and try connecting to each of those until one succeeds. 2022-01-05 19:03:41 +00:00
CalDescent
6041722250 Added missing import. 2022-01-05 18:35:03 +00:00
CalDescent
60d038b367 Return result of Network.connectPeer() back to caller. 2022-01-05 18:34:54 +00:00
CalDescent
b2c4bf96af Rebroadcast the entire list of hosted signatures each time our external IP address changes. 2022-01-03 19:52:42 +00:00
CalDescent
f007f9a86d Added optional "senderPeerAddress" string to HELLO messages, to allow external IP changes to be detected without using a centralized service. 2022-01-03 19:05:10 +00:00
CalDescent
b1c1634950 Updated log4j to 2.17.1
This involves modifying the log4j2.properties file on node startup to fix an incompatibility with ${dirname:-}. Thanks to AlphaX Projects for tracking down this incompatibility.
2022-01-02 20:50:38 +00:00
CalDescent
5157ccf7c0 Expire each authorization after 60 minutes. 2022-01-02 20:11:38 +00:00
CalDescent
c4a782301d Delete reader cache directories if a resource is unable to be built. 2022-01-02 18:31:17 +00:00
CalDescent
17fe94fa46 Added POST /render/authorize/{resourceId} endpoint
This allows an entire registered name to be preauthorized, therefore allowing for instance a website to automatically request other resources from the same author, such as videos.
2022-01-02 18:23:27 +00:00
CalDescent
75d9347d23 Show blank root page for the gateway.
This can ultimately be replaced with a website list / search engine.
2022-01-02 17:07:48 +00:00
CalDescent
ef784124f3 Fixed status bugs in loading screen. 2022-01-02 16:11:25 +00:00
CalDescent
bd1b631914 Use <base href="..."> in HTMLParser, rather than attempting to swap out every relative link
This delegates the task to the browser rather than doing it in java. It should also catch a few remaining types of links that we had missed - e.g. ones that originate from within js files.
2022-01-02 15:22:53 +00:00
CalDescent
edfc8cfdc4 Fixed bug which saved an incorrect peer address for a forwarded arbitrary signatures message. 2022-01-01 22:34:02 +00:00
CalDescent
fbe34015d4 Validate peer addresses before saving anything to the db. 2022-01-01 22:33:01 +00:00
CalDescent
391fa008d0 When making a direct connection to a peer, try using the default listen port along with the one specified in the ArbitraryPeers table.
This is a workaround to account for any ephemeral ports that may have made it into the dataset.
2022-01-01 21:37:23 +00:00
CalDescent
7df8381b8f Don't allow a row to be added to ArbitraryPeers (or the message to be rebroadcast) if an entry already exists with the same hash and host/ip.
This avoids duplicate entries from the same host/ip with differing ports. This can occur due to some requests using ephemeral port numbers. Ideally we would filter these out altogether, but this at least acts as a safety net to prevent a very cluttered db and associated "broadcast storm". The main tradeoff here is that multiple nodes on the same IP address will be recorded as a single entry. This doesn't seem like it will be a major limitation, because one of them will remain available.
2022-01-01 21:09:48 +00:00
CalDescent
c0234ae328 Added unit test to make sure the mempow nonce is being validated in ARBITRARY transactions. 2022-01-01 18:12:18 +00:00
CalDescent
5c64a85d7c Test a registered name with a space in it 2022-01-01 15:43:00 +00:00
CalDescent
7aa8f115ce Added "original copy indicator file", which prevents the node from deleting its own published content when storage space runs out.
Since some files won't have any mirrors, this prevents the cleanup manager from deleting the only copy in existence when freeing up space. This feature can be disabled by setting "originalCopyIndicatorFileEnabled": false in settings.json (or by deleting the ".original" files). The trade off is that the only copy in existence could be deleted if space gets low.

This will also allow for better reporting of own vs third party files in the local UI (not yet implemented).
2022-01-01 14:52:09 +00:00
CalDescent
cf2c8d6c67 Switched some IOExceptions to DataExceptions 2022-01-01 14:02:54 +00:00
CalDescent
37edebcad9 Fixed cleanup bug which could cause problems for ArbitraryDataWriter 2021-12-30 15:50:14 +00:00
CalDescent
4d4f661548 Renamed ArbitraryResourceSummary to ArbitraryResourceStatus, and added status titles & descriptions
This allows for consistent messaging about each status to be shown in different parts of the system. Previously these strings were hardcoded in the loading screen html so were inaccessible elsewhere.
2021-12-29 19:41:49 +00:00
CalDescent
46e4cb4f50 Removed custom version number 2021-12-29 12:47:49 +00:00
CalDescent
34e622cf0c Removed unused line. 2021-12-28 17:00:54 +00:00
CalDescent
7ccb99aa2c Improved clarity of file deletion message. 2021-12-28 17:00:41 +00:00
CalDescent
9e3847e56f Moved some of the less important arbitrary transaction related logs from INFO to DEBUG/TRACE.
Logs can be reinstated by adding these lines to log4j2.properties:

logger.arbitrary.name = org.qortal.arbitrary
logger.arbitrary.level = debug
logger.arbitrarycontroller.name = org.qortal.controller.arbitrary
logger.arbitrarycontroller.level = debug
2021-12-28 17:00:09 +00:00
CalDescent
90ced351f4 ARBITRARY transaction difficulty increased to 14, bringing it in line with MESSAGE transactions.
14 is currently the lowest accepted difficulty for on-chain transactions, so it's best that we don't go lower than that.
2021-12-28 14:25:17 +00:00
CalDescent
04295ea8c5 Set arbitrary transaction difficulty to 1 during unit tests, as they were taking too long. 2021-12-28 13:25:03 +00:00
CalDescent
2452d3c24b Moved proof of work difficulty definition from ArbitraryTransaction to ArbitraryDataManager 2021-12-28 13:01:26 +00:00
CalDescent
302428f1d1 Exclude RAW_DATA arbitrary transactions from various arbitrary data controller methods. 2021-12-28 12:01:03 +00:00
CalDescent
1f9f949a8c Added GET /peers/summary API which returns counts of the number of inbound and outbound connections that currently exist. 2021-12-24 15:25:58 +00:00
CalDescent
0bde1e97dc Added DELETE /resource/{service}/{name}/{identifier} API endpoint, to delete local data 2021-12-24 13:04:16 +00:00
CalDescent
42aca2e40f arbitraryDataCachedResources is now keyed by all ArbitraryDataResource elements, not just the resourceId 2021-12-24 12:12:16 +00:00
CalDescent
e1e44d35bb ArbitraryDataBuildQueueItem now extends ArbitraryDataResource
This is the start of a refactor to use ArbitraryDataResource objects rather than passing around separate resourceId, service and identifier, or other duplicate objects. Most of this will need to be done after the initial release due to time constraints.
2021-12-24 12:05:03 +00:00
QuickMythril
a790b2e529
reduce DOGE fees
https://github.com/dogecoin/dogecoin/blob/master/doc/fee-recommendation.md
2021-12-23 19:10:17 -05:00
CalDescent
357946388c New manager classes are no longer subclasses of Thread, as this part wasn't being used for anything. 2021-12-23 22:23:07 +00:00
CalDescent
b774583f28 Disabled all arbitrary data manager threads if QDN is disabled in the settings. 2021-12-23 22:19:14 +00:00
CalDescent
6436daca08 Split up ArbitraryDataManager into three separate manager classes, to make the code more readable and to keep each class within 500 lines.
This is still the least readable area of the data hosting code, but it should be a bit more maintainable now.
2021-12-23 22:15:28 +00:00
CalDescent
f153c7bb80 Added "qdnEnabled" setting (default: true) to allow users to opt out of QDN functionality. 2021-12-23 21:38:30 +00:00
CalDescent
1f8a618dcc Removed recently added logs. 2021-12-23 17:30:34 +00:00
CalDescent
2d853e5a2f Added support of patch creation from files without a newline at the end
The simplest solution was to only include a newline at the end of the patch file if the source file ended with a newline. This is used to inform the merge code as to whether to add the newline to the end of the resulting file. Without this, the checksums do not match (and therefore previously the complete file would have been included as a result).
2021-12-23 16:59:49 +00:00
CalDescent
361dc79ede Fixed bug caused by multiple concurrent builds of the same resource.
Several parts of the code request resources to be loaded/built, and these separate threads were tripping over each other and causing build failures. This has been avoided by making sure the resource isn't already building before requesting it.
2021-12-22 19:31:44 +00:00
CalDescent
19173321ea Small refactor to reduce code duplication. 2021-12-22 17:03:54 +00:00
CalDescent
87b724ec72 Updated various places that used File.separator when they could have used Paths.get() 2021-12-22 15:57:40 +00:00
CalDescent
67db0f950b Builds are now keyed by service, resourceId and identifier.
This allows for concurrent builds of multiple resources from the same registered name.
2021-12-22 15:44:00 +00:00
CalDescent
f85bbf12ca Applied reserved "default" keyword when publishing data too - it now maps to a blank identifier. 2021-12-22 13:59:08 +00:00
CalDescent
37e4f1e8d5 "default" is now a reserved keyword when used as an identifier.
Requesting a resource with the identifier "default" now maps to a blank string. This allows the /arbitrary/{service}/{name}/{identifier} endpoints to be used for default resources too, as they previously didn't support a blank string as the third parameter.
2021-12-22 13:33:08 +00:00
CalDescent
44d8bfd763 Added publish-auto-update-v5.pl - same as existing publish-auto-update.pl, but adapted to work with version 5 of ARBITRARY transactions.
The main differences are:
- Compute nonce instead of specifying a transaction fee
- Add blank/empty values for all the additional fields, as they are unused for auto updates
2021-12-22 13:00:29 +00:00
CalDescent
0cdbad6194 Bump version to 2.1.3-autoUpdateTest.0 - to check that auto updates still work with the QDN code. 2021-12-22 10:01:46 +00:00
CalDescent
4799a8a68e Don't allow files or file lists to be relayed if they relate to a blocked name.
This gives relay operators some control over the content they are relaying.
2021-12-22 09:21:08 +00:00
CalDescent
8caec81d1e Fixed content length bug 2021-12-21 16:15:20 +00:00
CalDescent
83d5bf45e5 Removed unnecessary call to isApiRestricted() 2021-12-21 16:04:32 +00:00
CalDescent
037eb8a163 Merge remote-tracking branch 'qortal/master' 2021-12-21 15:42:52 +00:00
CalDescent
3e5025b46e Domain map handling moved from RenderResource to a dedicated DomainMapResource
Including this as part of the rendering class was a bit of a hack; it's much cleaner to have a dedicated class.
2021-12-20 20:03:16 +00:00
CalDescent
623470209f Delete the combined file if has an incorrect hash after joining chunks together 2021-12-19 18:33:22 +00:00
CalDescent
553de5a873 Removed unnecessary javascript navigation code. 2021-12-19 17:25:58 +00:00
CalDescent
ccf8773b18 Added code that was left out from last commit. 2021-12-19 16:40:31 +00:00
CalDescent
cad25bf85d Add javascript to all HTML pages to allow for cross-origin navigation (back/forward buttons) from the UI.
Originally I had hoped to do this by using an intermediate iframe, to keep the message handlers separate from the user content, but this created CORS issues of its own. This approach is far simpler.
2021-12-19 12:21:29 +00:00
CalDescent
e9ab54f657 Fixed bug that prevented a resource from being overwritten if the existing data threw an exception when loading it. 2021-12-18 17:48:58 +00:00
CalDescent
0afb1a2d04 Make sure the blockchain is synced before running any data publishing code. 2021-12-18 16:07:29 +00:00
CalDescent
2d2b2964a5 When rebroadcasting an arbitrary signature list, don't send it back to the peer that we originally received it from. 2021-12-17 16:12:51 +00:00
CalDescent
10c4f7631b Added /zip versions of POST /arbitrary/* API endpoints.
These currently require the zip file's binary data to be encoded as base64, but it may not stay this way.
2021-12-17 14:28:07 +00:00
CalDescent
d921cffdaa Refactored ArbitraryResource to group by data type, to improve readability and code organization. 2021-12-17 12:50:00 +00:00
CalDescent
5369e21780 Allow the API whitelist to be easily disabled using the "apiWhitelistEnabled": false setting.
Now that we require API key authentication - and therefore security is greatly improved - many users will want to bypass the whitelist in order for the UI to communicate with their remote node. This gives an easy way to do this, without having to override the default whitelist. This boolean can now optionally be added to the default settings.json that is published with new releases, without removing the code's ability to update default whitelist values.
2021-12-17 10:22:08 +00:00
CalDescent
d34fb4494e Validate input data when uploading, to make sure it's not empty or missing. 2021-12-16 08:47:50 +00:00
CalDescent
1bd493ea37 Merge branch 'master' of github.com:Qortal/qortal
# Conflicts:
#	src/main/java/org/qortal/data/block/BlockData.java
2021-12-15 16:40:53 +00:00
CalDescent
3780767ccc Rotate hexagons by 90 degrees in the loading screen, to match the Qortal logo shape. 2021-12-15 13:27:42 +00:00
CalDescent
411279b3eb Removed all code relating to "fast sync" as it is not complete - we can reintroduce this later. 2021-12-15 13:27:12 +00:00
CalDescent
be3069e0e5 Only rebroadcast file list requests when in relay mode. 2021-12-15 12:29:05 +00:00
CalDescent
22cf870555 Updated some wording. 2021-12-15 12:16:11 +00:00
CalDescent
d6479c1390 ArbitraryDataFile.cleanupFilesystem() now uses generic FilesystemUtils.safeDeleteEmptyParentDirectories() code. 2021-12-15 12:15:58 +00:00
CalDescent
a4e82c79cc Completed implementation of relay mode
The procedure outlined in commit f4b06fb is now incorrect. Updated procedure:

- A node can opt into relay mode via the "relayModeEnabled":true setting.
- From this time onwards, they will ask their peers if they ever receive a file list request that they cannot serve by themselves.
- Whenever a peer responds with a file list, it is forwarded on to the originally requesting peer, complete with the peer address of the node that responded. Currently, only the first response is forwarded, but we may later decide to forward all responses.
- As well as forwarding, the relay peer keeps track of the peers that report to be holding hashes (these mappings are held for 30 seconds).
- The originally requesting peer can then make a request to the relay peer for the data file(s).
- The relay peer uses the mapping to forward the request on to another peer, and then forwards the response (i.e. the data file) back to the peer that originally requested the file.
2021-12-15 12:15:31 +00:00
CalDescent
bcc89adb5f Removed "peerAddress" from ArbitraryDataFileListMessage (introduced in recent commit)
It's best that the source peer's address isn't exposed to the requesting peer. The relay peer can keep track of this mapping itself.

The only real issue with this approach is that we can't use data from ArbitraryDataFileListMessage to update our ArbitraryPeers data, because we can't distinguish between relay peers and hosting peers. But this isn't something we currently do anyway, as we have the ARBITRARY_SIGNATURES message type to take care of updating ArbitraryPeers mappings.
2021-12-15 11:40:54 +00:00
CalDescent
a41c9e339a Fixed occasional NPE 2021-12-15 11:31:12 +00:00
CalDescent
feeca77436 Added directDataRetrievalEnabled setting (default true)
Setting this to false prevents new connections being made to peers that report to have the data that is needed. This is likely only useful for testing, as disabling it in production would reduce the success rate of data retrieval.
2021-12-15 11:29:49 +00:00
CalDescent
fcce12ba40 Added missing code from last commit. 2021-12-15 11:28:34 +00:00
CalDescent
f4b06fb834 Added relay mode for file list requests
This reuses most of the code already in place in the core related to forwarding.

- A node can opt into relay mode via the "relayModeEnabled": true setting
- From this time onwards, they will ask their peers if they ever receive a file list request that they cannot serve by themselves
- Whenever a peer responds with a file list, it is forwarded on to the originally requesting peer, complete with the peer address of the node that responded
- The original peer can then make a request for the data file(s) themselves using a similar approach, specifying the IP address of the ultimate peer so that the relay node knows who to ask. This part is not implemented yet.
2021-12-12 10:51:24 +00:00
CalDescent
cc297ccfcd Removed the word "service" from all settings, to avoid confusion with arbitrary data services.
Updated setting names:

domainMapServiceEnabled -> domainMapEnabled
domainMapServicePort -> domainMapPort
gatewayServiceEnabled -> gatewayEnabled
gatewayServicePort -> gatewayPort
2021-12-10 12:29:33 +00:00
CalDescent
c7c88dec04 Attempt to differentiate between resources that are downloading, and ones where downloading hasn't been attempted yet. 2021-12-10 12:22:26 +00:00
CalDescent
e481a5926a Use "blocked" instead of "blacklisted", for consistency with the buttons and terminology in the UI 2021-12-10 11:34:10 +00:00
CalDescent
0464245218 Simplified lists feature to be referenced by a single name, instead of the confusing "category" and "resourceName" properties.
This makes them extremely generic, improves filenames, and makes it easier to create custom lists. It doesn't have backwards support, but the lists feature isn't working properly in core 2.1+ anyway.
2021-12-10 11:33:26 +00:00
CalDescent
0001f31c06 Merge remote-tracking branch 'qortal/master' 2021-12-10 09:35:44 +00:00
CalDescent
2efac0c96b Modified zip implementation to preserve filenames of single file resources.
Also modified the directory structure of single file resources to make them consistent with multi file resources.

For multi file resources, the original folder is renamed to "data", resulting in a layout such as:
data/file1.txt
data/file2.txt
data/dir1/file3.txt

For single file resources, the file is now moved into a "data" folder, like so:
data/file.txt

This is slightly unconventional, but is appropriate within the context of QDN to keep everything consistent.
2021-12-06 19:53:20 +00:00
CalDescent
19240a9caf Removed duplicated strings in zip tests 2021-12-05 18:13:02 +00:00
CalDescent
4eef28f93d Added unit tests for zipping / unzipping 2021-12-05 17:38:53 +00:00
CalDescent
c6e5c4e3b5 Tidy up of storage thresholds and comments. Also reduced deletion threshold from 100% to 98% to reduce the chances of filling up the disk and corrupting the db. 2021-12-05 16:36:54 +00:00
CalDescent
007f567c7a Improved directory cleanup process to avoid leaving empty parent directories lying around. 2021-12-05 16:24:47 +00:00
CalDescent
ffe178c64c Delete directories in the data folder that have no associated transaction.
This adds support for "unconfirmable" data uploads, which will be useful for Q-Chat. It also handles cases where a transaction is orphaned and then subsequently becomes invalid.
2021-12-05 16:24:21 +00:00
CalDescent
c3835cefb1 Fixed bug relating to storage manager instances. 2021-12-05 14:42:12 +00:00
CalDescent
2c382f3d3f Fixed logging error. 2021-12-05 13:30:22 +00:00
CalDescent
b592aa6a02 Added custom validation for websites.
A website must contain one of the following files in its root directory to be considered valid:

index.html
index.htm
default.html
default.htm
home.html
home.htm

This is the first page that is loaded when loading a Qortal-hosted website.
2021-12-05 13:30:10 +00:00
CalDescent
57e82b62a1 Increased the capabilities of the service validation functions. 2021-12-05 13:03:22 +00:00
CalDescent
94b17eaff3 Added unit test for random file deletion, and fixed some issues found via the test. 2021-12-04 16:36:20 +00:00
catbref
eb9b94b9c6 Add Qortal AT FunctionCodes for getting account level / blocks minted + tests 2021-12-04 16:36:05 +00:00
CalDescent
a3038da3d7 Moved some shared arbitrary test methods to a new ArbitraryUtils class. 2021-12-04 14:26:10 +00:00
CalDescent
6026b7800a Fixed some warnings. 2021-12-04 14:23:21 +00:00
CalDescent
36c5b71656 Delete data associated with a name at random, if a name is using more than its allocated limit.
This would happen if a name fills their limit, and then additional names are followed. Alternatively it could happen if the total storage capacity reduces due to disk space being used by other apps. Chunks are deleted at random to reduce the chance of the same chunk being deleted everywhere. Data loss is possible here for transactions that don't have many peers. We'll have to see in practice how much of a problem this is, but it's better than the scenario where one content creator consumes all space on their followers' nodes, leaving no space for other names that are subsequently followed.
2021-12-04 14:23:09 +00:00
CalDescent
a320bea68a Limit the amount of data that can be stored per name.
This is calculated by the total capacity divided by the number of names the node follows. The idea here is that a single content creator can't upload terabytes of data and consume all the space on their followers' nodes. They can only use a proportion, with equal space given to each followed name. And since the limit is dynamic, following more names reduces the allocation to existing names.
2021-12-04 13:33:45 +00:00
CalDescent
a87fe8b44d Combined path filtering into a single filter to avoid iterating through the list multiple times. 2021-12-04 12:06:15 +00:00
CalDescent
0a2b4dedc7 Don't sort the transactions in listAllHostedTransactions() as this is a waste of CPU. 2021-12-04 12:03:52 +00:00
CalDescent
f7ed3eefc8 Verify that each data file matches the size reported by transaction.
This discourages an incorrect file size being included with a transaction, as the system will reject it and won't even serve it to other peers.

FUTURE: we could introduce some kind of blacklist to track invalid files like this, and avoid repeated attempts to retrieve them. It is okay for now as the system will backoff after a few attempts.
2021-12-04 12:00:38 +00:00
CalDescent
8bb3a3f8a6 Added hosted transactions cache
This skips the need to check the filesystem if no new files have arrived since the last lookup.
2021-12-04 09:28:45 +00:00
CalDescent
89d08ca359 Renamed method from listAllHostedData() to listAllHostedTransactions() 2021-12-04 09:27:39 +00:00
CalDescent
b80aec37e0 Added GET /arbitrary/hosted/resources API to list all arbitrary resources that the node is hosting (at least partial) data for. 2021-12-03 21:43:18 +00:00
CalDescent
e34fd855a9 Added GET /arbitrary/hosted/transactions API to list all arbitrary transactions that the node is hosting (at least partial) data for.
This API call could get quite heavy when large amounts of files are hosted, but it's preferable to maintaining a list in the database. Ideally we need to keep the database generic so that it can be bootstrapped without interfering with the state. We can always add caching and rate limiting if needed.
2021-12-03 21:11:56 +00:00
CalDescent
fc12ea18b8 Fixed bug in getArbitraryResources() 2021-12-03 18:17:20 +00:00
CalDescent
f87df53791 Don't compute a nonce when creating arbitrary transactions. Instead this can be done by calling POST /arbitrary/compute, or more ideally by the client, such as the UI. 2021-12-03 17:33:07 +00:00
CalDescent
d6746362a4 Reduced log spam when a file can't be served. 2021-12-02 19:41:31 +00:00
CalDescent
2850bd0b46 Added new GET /arbitrary/resources/names endpoint to fetch resources grouped by name. 2021-12-02 19:41:07 +00:00
CalDescent
b762eff4eb Announce to the network when uploading a new file.
Previously only subsequent peers would announce, not the initial uploader. This made it very difficult to locate brand new files.
2021-12-01 20:42:51 +00:00
CalDescent
4b3b96447f Require an API key or prior authorization on GET /arbitrary/resource/status/* endpoints 2021-12-01 20:03:36 +00:00
CalDescent
13bcfbe3c5 Fixed issues preventing the loading screen from working when using the gateway. 2021-12-01 19:51:45 +00:00
CalDescent
8525fb89f8 Use a zero fee for ARBITRARY transactions, as we require a PoW nonce instead. 2021-12-01 18:12:33 +00:00
CalDescent
ed2d1c4932 Improved logging 2021-12-01 16:14:24 +00:00
CalDescent
5091f8457e Terminate metadata files with a newline 2021-12-01 16:13:59 +00:00
CalDescent
84b69fc58c Revert "Use response code 102 ("Processing") instead of 503 ("Service Unavailable") for the loading screen"
This reverts commit 8823f69256aaa2155987a53de5153dfe5f09a2c9.
2021-12-01 13:56:47 +00:00
CalDescent
a2cac003a4 Major rework of chunk hashes
Chunk hashes are now stored off chain in a metadata file. The metadata file's hash is then included in the transaction.

The main benefits of this approach are:
1. We no longer need to limit the total file size, because adding more chunks doesn't increase the transaction size.
2. This increases the chain capacity by a huge amount - a 512MB file would have previously increased the transaction size by 16kB, whereas it now requires only an additional 32 bytes.
3. We no longer need to use variable difficulty; every transaction is the same size and so the difficulty can be constant no matter how large the files are.
4. Additional metadata (such as title, description, and tags) can ultimately be stored in the metadata file, as apposed to using a separate transaction & resource.
5. There is also scope for adding hashes of individual files into the metadata file, if we ever wanted to allow single files to be requested without having to download and build the entire resource. Although this is unlikely to be available in the short term.

The only real negative is that we now how to fetch the metadata file before we know anything about the chunks for a transaction. This seems to be quite a small trade off by comparison.

Since we're not live yet, there is no backwards support for on-chain hashes, so a new data testchain will be required. This hasn't been tested outside of unit tests yet, so there will likely be several fixes needed before it is stable.
2021-12-01 12:37:21 +00:00
CalDescent
7c16a90221 Moved relocation code from isDataLocal() to onImportAsUnconfirmed()
It's not good to be moving files around in a method that should really be read only. This also adds an intentional checkAndRelocateMiscFiles() call rather than relying on a call to isDataLocal() which may be removed at any time.
2021-12-01 12:13:23 +00:00
CalDescent
97cdd53861 Fixed bugs in safeDeleteEmptyParentDirectories() 2021-12-01 11:59:14 +00:00
CalDescent
b7ee00fb22 Fixed errors in Admin API tests due to failing authentication. 2021-11-27 20:08:59 +00:00
CalDescent
ef2ee20820 Merge remote-tracking branch 'qortal/master'
# Conflicts:
#	pom.xml
#	src/main/java/org/qortal/api/resource/ListsResource.java
#	src/main/java/org/qortal/list/ResourceList.java
#	src/main/java/org/qortal/list/ResourceListManager.java
#	src/main/java/org/qortal/transaction/ChatTransaction.java
2021-11-27 19:41:17 +00:00
CalDescent
4866e5050a If a single file resource is being published and a complete file patch has been chosen, make sure to use PUT instead of PATCH as there's nothing to be gained by adding another layer.
This would have been caught by the max differences check anyway, but it's a good check to have in place in case we recalibrate or remove the differences check in the future.
2021-11-27 19:35:22 +00:00
CalDescent
8e36c456e1 Wait for storage space to be calculated before running storage policy tests. 2021-11-27 18:06:48 +00:00
CalDescent
4b8bcd265b Make sure unit test use a different lists directory, and delete it before and after each test. 2021-11-27 18:05:25 +00:00
CalDescent
0db681eeda Fixed failing storage policy tests due to not calculating the available storage 2021-11-27 17:56:34 +00:00
CalDescent
8823f69256 Use response code 102 ("Processing") instead of 503 ("Service Unavailable") for the loading screen 2021-11-27 16:48:47 +00:00
CalDescent
f3e9dfe734 Return a 404 instead of a 500 if a resource is unavailable.
Could be improved in the future to return different codes depending on its status (e.g. doesn't exist = 404, 102 for loading, 500 for error, etc), but 404 makes the most sense until that has been developed
2021-11-27 16:43:29 +00:00
CalDescent
a7b31ab1f9 Small bug fixes 2021-11-27 16:40:12 +00:00
CalDescent
644ab27186 Updated wording 2021-11-27 14:22:09 +00:00
CalDescent
e90ecd2085 Adapted GET /arbitrary/resources endpoint to allow filtering by identifier
- If an identifier parameter is missing or empty, it will return an unfiltered list of all possible identifiers.
- If an identifier is specified, only resources with a matching identifier will be returned.
- If default is set to true, only resources without identifiers will be returned.
2021-11-27 14:21:36 +00:00
CalDescent
bc38184ebf Major rework of local data directory structure
Files are now keyed by signature, in the format:
data/si/gn/signature/hash

For times when there is no signature available (i.e. at the time of initial upload), files are keyed by hash, in the format:
data/_misc/ha/sh/hash

Files in the _misc folder are subsequently relocated to a path that is keyed by the resulting signature.

The end result is that chunks are now grouped on the filesystem by signature. This allows more transparency as to what is being hosted, and will also help simplify the reporting and management of local files.
2021-11-27 13:00:32 +00:00
CalDescent
0dd43d5c9a Fixed bug in storage calculation 2021-11-24 14:22:57 +00:00
CalDescent
e879bd0fc5 Delete some random chunks when we reach the storage capacity
This should allow for a relatively even distribution of chunks, but there is a (currently unavoidable) risk of files with very few mirrors being deleted altogether.

Longer term this could be improved by checking that one of our peers has a file, before it's deleted locally
2021-11-24 14:15:22 +00:00
CalDescent
8bf7daff65 Track the storage capacity and the total data/temp directory sizes
Nodes will stop proactively storing new data when they reach 90% capacity.

A new "maxStorageCapacity" setting has been added to allow the user to optionally limit the allocated space for this node. Limits are approximate only, not exact.
2021-11-24 13:43:45 +00:00
CalDescent
ae0f01d326 Added storage policy unit tests 2021-11-24 11:02:54 +00:00
CalDescent
af8d0a3965 Separated computeNonce() from build() in the transaction builder.
This gives the option of the nonce to be computed elsewhere, such as in the UI, and also allows transaction unit tests to run much more quickly.
2021-11-24 11:02:17 +00:00
CalDescent
1b170c74c0 Modified storage code to support 2 new settings:
publicDataEnabled - whether to store decryptable data (default true)
privateDataEnabled - whether to store data without a decryption key (default false)
2021-11-24 09:38:18 +00:00
CalDescent
f6b9ff50c3 More loading screen improvements 2021-11-23 22:21:57 +00:00
CalDescent
9ef75ebcde Improved styling of loading panel 2021-11-23 21:15:45 +00:00
CalDescent
f76a618768 Display the latest status on the loading screen, updated via API calls on a timer 2021-11-23 20:53:09 +00:00
CalDescent
ec48ebcd79 Improved resource statuses 2021-11-23 09:14:44 +00:00
CalDescent
908f80a15d Fixed bug when checking if all files exist locally in /arbitrary/status 2021-11-22 08:43:07 +00:00
CalDescent
02eab89d82 Fixed bug when trying to delete a file instead of a directory. 2021-11-21 19:24:20 +00:00
CalDescent
c588786a06 Added /base64 variation of POST /arbitrary/* APIs
This can be used to upload base64 encoded file data directly from the UI. Using base64 because base58 is unusably slow
2021-11-21 19:12:01 +00:00
CalDescent
b4f3105035 Added /render/authorize/{service}/{resourceId}* APIs
These allow the UI to pre-authorize a resource and therefore avoid having to pass a sensitive API key to a website or app.
2021-11-21 14:57:26 +00:00
CalDescent
d0000c6131 If "build=true" is specified in query string of GET /resource/status/{service}/{name}, build the resource before returning the status 2021-11-20 18:52:03 +00:00
CalDescent
c05ffefd7d Added a dynamic background to the loading screen. Still needs a lot of work. 2021-11-20 14:09:00 +00:00
CalDescent
530fc67a05 List available services in API docs when requesting arbitrary data 2021-11-20 11:46:03 +00:00
CalDescent
c79ec11b07 Fixed incorrect comment 2021-11-19 23:16:28 +00:00
CalDescent
668ef26056 Fixed major performance issue with GET /arbitrary/resources?includestatus=true
The missing data check was triggering decryptions, extractions, etc. Replaced with some code which checks for the presence of chunks on the local machine, without getting involved with any build process overhead.
2021-11-19 23:00:36 +00:00
CalDescent
75ec7723ef Improved accuracy of statuses
- "NOT_STARTED" is now "DOWNLOADED"
- "DOWNLOADING" is now "MISSING_DATA"
- Removed "DOWNLOAD_FAILED"

Some of these could be reintroduced once the system is able to support them.
2021-11-19 21:48:41 +00:00
CalDescent
73e609fa29 Replaced all IllegalStateException with DataException in arbitrary code
This was necessary to ensure that all exceptions are caught intentionally, as otherwise it creates endless amounts of edge cases.
2021-11-19 21:42:03 +00:00
CalDescent
8cb06bf451 Return statuses in GET /arbitrary/resources endpoint when the "includestatus" parameter is true. 2021-11-19 20:20:45 +00:00
CalDescent
1be8a059f4 Fixed bug caused by not catching a IllegalStateException
I may replace all IllegalStateExceptions with DataExceptions soon so we can ensure they are always caught.
2021-11-19 20:18:56 +00:00
CalDescent
7f41c7ab0e Added "BLACKLISTED" status for arbitrary data resources. 2021-11-19 20:18:00 +00:00
CalDescent
3860c5d8ec Fixed some failing tests. 2021-11-19 16:12:31 +00:00
CalDescent
a061a7cc4d Fixed various warnings raised by the IDE. 2021-11-19 16:11:37 +00:00
CalDescent
844501d6cd Added GET /arbitrary/resource/status/* API endpoints
These can be used to check the current status of a resource. The different statuses are:

NOT_STARTED,
DOWNLOADING
DOWNLOADED
BUILDING
READY
DOWNLOAD_FAILED
BUILD_FAILED
UNSUPPORTED

Not all statuses are returned yet. The build process needs more functionality to be able to support DOWNLOADED and DOWNLOAD_FAILED. Also, BUILDING and BUILD_FAILED are currently unable to distinguish between different resources with the same registered name, so need some attention.
2021-11-19 15:26:52 +00:00
CalDescent
020bd00b8f Removed incorrect @SecurityRequirement annotation 2021-11-19 13:40:02 +00:00
CalDescent
0706b0d287 Added some /site endpoints to the gateway, for backwards support of the demo sites 2021-11-19 13:26:47 +00:00
CalDescent
ce56cd2b16 Disallow local (loopback address) requests when using the gateway
This removes the possibility of some locally running javascript in a website or app requesting unvetted data via the local gateway.
2021-11-19 13:20:53 +00:00
CalDescent
b7a0a7eea4 Removed API authentication when using the gateway, as that would defeat the purpose of it. 2021-11-19 13:14:57 +00:00
CalDescent
824d14e793 Removed unnecessary check for isApiRestricted() when previewing.
The API key authentication will be enough to restrict requests.
2021-11-19 13:08:56 +00:00
CalDescent
83e0ed2b5d Reduced log spam 2021-11-19 12:59:25 +00:00
CalDescent
c8b70b51c3 Added gateway service, to allow websites to be served directly on a domain or IP.
This replaces the existing GET /site/{name} API endpoints.

Example settings:

"gatewayServiceEnabled": true,
"gatewayServicePort": 80

Websites can then be served using URL:

http://localhost/RegisteredName

Or, if node is behind public DNS:

http://example.com/RegisteredName

Or, if a custom port (such as 12393) is used:

http://localhost:12393/RegisteredName
http://example.com:12393/RegisteredName

This is currently for serving websites only, but can be adapted to serve other services if needed.
2021-11-19 12:59:15 +00:00
CalDescent
c0fedaa3a4 Attempt to request files directly from a peer if it isn't returned in the general network broadcast. 2021-11-19 12:05:40 +00:00
CalDescent
e74dcff010 Wait for 3 seconds between attempts to allow time for data to arrive. 2021-11-19 11:21:40 +00:00
CalDescent
3b5b45b463 Give up after 5 attempts to request data in the GET /arbitrary APIs 2021-11-19 11:21:14 +00:00
CalDescent
fead482b0d Fixed bugs introduced in preview functionality. 2021-11-17 19:32:35 +00:00
CalDescent
29bd8203b5 Removed service from POST /render/preview/{service} API as it isn't needed right now 2021-11-17 19:25:21 +00:00
CalDescent
08b79e45cf /site API endpoints replaced with more generic /render APIs so that they can be used for apps, blogs, etc
This involves passing a service along with the name, such as `GET /render/WEBSITE/Test`
2021-11-17 19:22:25 +00:00
CalDescent
3a05a0bcaa Added "LIST" service - to be used to optionally host an otherwise private list if the user wants to share it publicly. 2021-11-17 18:59:45 +00:00
CalDescent
d0aafaee60 Added POST /arbitrary/../string API endpoints to allow data to be passed to the core as a string.
This will be useful for metadata, playlists, etc, as well as some types of data published by Qortal apps.
2021-11-17 18:57:46 +00:00
CalDescent
332b874493 Removed /arbitrary PUT and PATCH API endpoints.
It's best to let the core decide which one to use now that it is able to.
2021-11-16 19:36:24 +00:00
CalDescent
6c995ed738 Validation removed from METADATA so that it is more generic - it's up to each application to decide how to structure its contents. Existing strict validation applied to a duplicate called QORTAL_METADATA. This will be the one used for website/app listings in the Qortal UI. 2021-11-16 19:32:14 +00:00
CalDescent
fb09d77cdc Rework of "Service" types to allow for validation
Each service supports basic validation params, plus has the option for an entirely custom validation function.

Initial validation settings:
- IMAGE must be less than 10MiB
- THUMBNAIL must be less than 500KiB
- METADATA must be less than 10KiB and must contain JSON keys "title", "description", and "tags"
2021-11-16 19:28:25 +00:00
CalDescent
9c952785e6 Allow the API key to be passed as an "apiKey" parameter in the query string or POST body as an alternate option to a header.
This is needed to avoid triggering a CORS preflight (which occurs when using an X-API-KEY header). The core isn't currently capable of responding to a preflight and the UI therefore blocks the entire request. See: https://stackoverflow.com/a/43881141
2021-11-14 20:24:02 +00:00
CalDescent
2f51c1bf47 Added all missing SecurityRequirement annotations 2021-11-14 19:56:26 +00:00
CalDescent
276a110e90 Fixed bug in API key comparison 2021-11-14 19:55:58 +00:00
CalDescent
b761674b2c Default temp path moved to a subfolder of the data path
This allows users to set only their data path, and for the temp folder to automatically follow it. The temp folder can be moved to a custom location by setting the "tempDataPath" setting.
2021-11-14 17:00:49 +00:00
CalDescent
0b20bf0145 Website serving now requires authentication for everything except the "domain map" server 2021-11-14 16:00:58 +00:00
CalDescent
1397cbeac2 General API key / security-related updates 2021-11-14 15:59:08 +00:00
CalDescent
06e122f303 Added 'localAuthBypassEnabled' setting to allow users to opt in to the old method of local authentication at their own risk. 2021-11-14 15:24:15 +00:00
CalDescent
f062acfd7c Rework of API keys
An API key is now _required_ for sensitive API calls that would previously have allowed local loopback authentication.

Previously, a request would have been considered authenticated if it originated from the same machine, however this creates a security issue when running third party code (particularly javascript) via the data network.

The solution is to now require an API key to authenticate sensitive API calls no matter where the request originates from.

It works as follows:

- When the core is first installed, it has no API key generated and will block sensitive calls until generated.
- A new POST /admin/apikey/generate API endpoint has been added, which can be used the generate an API key for a newly installed node. The UI will ultimately call this automatically.
- This API returns the generated key so that it can be stored by the requesting app (most likely the UI).
- From then on, the generate API requires authentication via the existing API key in order to regenerate a key. It can be used as a security measure if the existing key is compromised.
- The API key must be passed to all sensitive API endpoints from then on, even when calling it from the same local machine.
- If the core already has a legacy API key specified via the 'apiKey' setting, this will be automatically copied to the new format so that a new one doesn't need to be generated.
- The API key itself is stored in a flat file in the qortal directory (the path can be customized using the `apiKeyPath` setting). Deleting this file and restarting the core will allow a new one to be regenerated.
2021-11-14 15:14:37 +00:00
CalDescent
97ca414fc0 Revert "Added "apiKeyDisabled" setting to bypass API key / loopback checking for those who need it."
This reverts commit 8a7446fb40e8177c8493c43c6a340d6d55ddd590.
2021-11-13 19:19:54 +00:00
CalDescent
a9af5bcec4 Website serving code moved to a new class called ArbitraryDataRenderer
The process of serving resources to a browser will likely be needed for more than just websites (e.g. it will be needed for apps too) so it makes sense to abstract it to its own class.
2021-11-13 19:09:35 +00:00
CalDescent
7e30bf4197 Fixed website preview functionality which isn't compatible with asynchronous building.
The simplest solution was to build synchronously when previewing.
2021-11-13 17:40:09 +00:00
CalDescent
c724ea9f69 Removed various /arbitrary API endpoints that were only really useful at the start of the data storage project. 2021-11-13 17:11:40 +00:00
CalDescent
e6cc4a1180 Improved logging for times when data requests are rate limited. 2021-11-13 16:59:39 +00:00
CalDescent
3cce097b9d When a newer PUT exists for a resource, delete records of peers holding earlier transactions
This should help keep the peer lookup table size down, as there is no need to locate files for transactions that existed before the most recent PUT transaction.
2021-11-13 16:58:23 +00:00
CalDescent
53f9d6869d Improved logging when a resource has no identifier 2021-11-13 16:52:02 +00:00
CalDescent
61beee0f49 Tidied up unfinished arbitrary data payments code. 2021-11-13 15:00:52 +00:00
CalDescent
1f3d400ad6 Small refactor of previous commit's code to improve consistency. 2021-11-13 13:47:29 +00:00
CalDescent
f2ff2187d9 Case sensitivity preference can now be specified when checking if an item is in a list.
All registered name checks are now case insensitive, since the names themselves are case insensitive.
2021-11-13 13:37:16 +00:00
CalDescent
28ddc0055f Implemented reader cache clearing
Built resources are deleted when either:
- The resource reaches the expiry interval specified in the builtDataExpiryInterval setting (default 30 days)
- The resource is published by a name that is in the local blacklist

Resources only exist in the reader cache once they have been viewed, to remove the loading time on subsequent views. But some may prefer to reduce this expiry time (at the expense of longer load times and more CPU), as data is held unencrypted in the cache.
2021-11-13 13:35:40 +00:00
CalDescent
90b5b6bd8b Don't allow data to be fetched for viewing if the name is in the local blacklist.
We still allow it to be fetched even if it's outside of the storage policy, as the cleanup manager will delete the files very soon after, and they won't be allowed to be served to other peers due to other checks already in place.
2021-11-13 12:54:14 +00:00
CalDescent
53466797a5 Once we receive a file from a peer, add the mapping to the lookup table.
This allows other peers to find out where they can obtain these files if we were to stop hosting them later. Or even if we continue hosting copies, it still informs the network on other locations, for better decentralization.
2021-11-13 12:50:26 +00:00
CalDescent
f5235938b7 Rate limit any file list broadcasts
We don't want the network being spammed when a file isn't available by any reachable peers. This feature ensures retries are spaced out over longer timeframes. Basic logic:

- Wait 5 minutes in between failed attempts
- After 5 failed attempts (i.e. 25 mins) only try once per day from then on
- A core restart resets the counters

The stats gathered here can also be used to inform the core of when it should attempt a direct connection with a peer to obtain the data. That part isn't implemented yet.
2021-11-13 12:26:27 +00:00
CalDescent
054860b38d Rework of storage policy handling, as the previous implementation didn't handle viewed data properly.
This also adds a feature to allow data to be deleted and no longer served once a name has been blacklisted.
2021-11-13 11:35:29 +00:00
CalDescent
b60d02b8f4 Fixed preexisting list name issue when blocking chat transactions by address 2021-11-13 11:13:19 +00:00
CalDescent
0d69797851 Block chat transactions on the local node if its sender owns a name that is blacklisted by the user. 2021-11-13 11:12:52 +00:00
CalDescent
bfffff0750 /lists APIs now made fully generic
This allows for custom list creation without the need for creating API endpoints to go along with it. This should save time now that we are using lists more.
2021-11-13 11:00:01 +00:00
CalDescent
b7bcd8da7d Prevent arbitrary data transactions being created unless the network supports it (i.e. the hard fork has taken place). 2021-11-13 10:29:01 +00:00
CalDescent
d3862c97ba Added "APP" and "METADATA" service types
- "APP" will allow for user-created apps and the Qortal app store
- "METADATA" will be used to supply info about apps/websites/resources, such as title, description, tags, etc
2021-11-13 10:06:53 +00:00
CalDescent
c069c39ce1 Implemented automatic PUT/PATCH detection
When using POST /arbitrary/{service}/{name}... it will now automatically decide which method to use (PUT/PATCH) based on a few factors:

- If there are already 10 or more layers, use PUT to reset back to a single layer
- If the next layer's patch is more than 20% of the total resource file size, use PUT
- If the next layer modifies more than 50% of the total file count, use PUT
- Otherwise, use PATCH

The PUT method causes a new base layer to be created and all previous update history for that resource becomes obsolete. The PATCH method adds a small delta layer on top of the existing layer(s).

The idea is to wipe the slate clean with a new base layer once the patches start to get demanding for the network to apply. Nodes which view the content will ultimately have build timeouts to prevent someone from deploying a resource with hundreds of complex layers for example, so this approach is there to maximize the chances of the resource being buildable.

The constants above (10 layers, 20% total size, 50% file count) will most likely need tweaking once we have some real world data.
2021-11-13 09:56:13 +00:00
CalDescent
e994d501b0 Fixed errors in documentation 2021-11-12 18:22:10 +00:00
CalDescent
caf163f98c Include "tempDataPath" in test settings so that tests don't put files in the main temp directory. 2021-11-12 17:46:48 +00:00
CalDescent
1c408db907 Rework of arbitrary APIs and qdata to support identifiers
qdata has reached the stage of needing parameterized arguments, but this is low priority now that we have data functionality within the UI itself.
2021-11-12 17:42:21 +00:00
CalDescent
8d44e07c32 Fixes issues relating to reading resources containing a single file 2021-11-12 17:37:33 +00:00
CalDescent
d99fae4340 Added support for single file patching
This ensures that the folder structures align when comparing before and after versions.
2021-11-12 17:37:02 +00:00
CalDescent
d49caa29ce Pass Service enum to TransactionRepository.getSignaturesMatchingCriteria() instead of an Integer.
This fixes a bug when no service was specified in the /arbitrary/search API.
2021-11-12 14:22:22 +00:00
CalDescent
8bebe11b4e Allow single files to be uploaded without compression
We may choose to save on CPU by not compressing individual files, so this allows the network to support that. However it is still using compression by default, to reduce file sizes.
2021-11-12 13:44:28 +00:00
CalDescent
236a456cae Added support for single file uploads.
This process could potentially be simplified if we were to modify the structure of the actual zipped data (on the writer side), but this approach is more of a "catch-all" (on the reader side) to support multiple different zip structures, giving us more flexibility. We can still choose to modify the written zip structure if we choose to, which would then cause most of this new code to be skipped.

Note: the filename of a single file is not currently retained; it is renamed to "data" as part of the packaging process. Need to decide if this is okay before we go live.
2021-11-12 13:35:50 +00:00
CalDescent
7bc745fa8e Added "THUMBNAIL" and "PLAYLIST" service types, and fixed a duplicate issue in earlier commit.
Thumbnails will be used in order to show logos/screenshots in the list of websites or other resources. Playlists will allow for media apps to group videos/audio/images into collections, e.g. albums.
2021-11-12 09:02:44 +00:00
CalDescent
056fc8fbaf Treat a blank identifier as null 2021-11-12 08:59:43 +00:00
CalDescent
b6aa507b41 Added "AUDIO" and "BLOG" service types.
BLOG_POST and BLOG_COMMENT are using values 777 and 778 as these were the values used in Qora.
2021-11-11 09:16:16 +00:00
CalDescent
4b1a5a5e14 Connected the rest of the system up to the recently added "identifier" feature. 2021-11-11 09:12:54 +00:00
CalDescent
a364206159 Added "IMAGE", "VIDEO" and "DOCUMENT" service types. 2021-11-07 18:44:05 +00:00
CalDescent
b5feb5f733 Fixed test which was failing due to an earlier commit 2021-11-07 18:41:52 +00:00
CalDescent
991125034e Added "identifier" property to arbitrary transactions
Until now we have been limited to one data resource per name/service combination. This meant that each name could only have a single website, git repo, image, video, etc, and adding another would overwrite the previous data. The identifier property now allows an optional string to be supplied with each resource, therefore allowing an unlimited amount of resources per name/service combination.

Some examples of what this will allow us to do:

- Create a video library app which holds multiple videos per name
- Same as above but for photos
- Store multiple images against each name, such as an avatar, website thumbnails, video thumbnails, etc. This will be necessary for many "system level" features.
- Attach multiple websites to each name. The default website (with blank/null identifier) would remain the entry point, but other websites could be hosted essentially as subdomains, and then linked from the default site. This also provides a means to go beyond the 500MB website size limit.

Not all of these features will exist initially, but having this identifier included in the protocol layer allows them to be added at any time.
2021-11-07 18:39:43 +00:00
CalDescent
a0fe1a85f1 Removed website publishing API since we now do everything using POST /arbitrary/{service}/{name} 2021-11-07 18:23:14 +00:00
CalDescent
3a2e68c334 Improved directory structure of the "reader" cache 2021-11-07 17:16:42 +00:00
CalDescent
b6418cd912 Revert "Added an index to help speed up recent queries"
This reverts commit e652038018a3a0d24ecafa3b9c11d82489f05e5c.
2021-11-06 12:52:20 +00:00
CalDescent
e652038018 Added an index to help speed up recent queries 2021-11-06 12:47:32 +00:00
CalDescent
b2e2af51ed Added API endpoint to list all arbitrary resources, grouped by name and service
This is used by the UI to list available websites (and ultimately other categories of hosted data)
2021-11-06 10:41:30 +00:00
CalDescent
a0fe803c35 Added POST /arbitrary/{service}/{name} API endpoint
At the moment this just redirects to PUT, but will ultimately choose PUT or PATCH based on the differences in the data supplied.
2021-11-05 14:46:22 +00:00
CalDescent
ea2ca37abe Improved format of error messages 2021-11-05 14:38:10 +00:00
CalDescent
0601ffbb34 Added /transactions/convert method to use to convert full transaction bytes to ones used for signing
This is needed in order for the UI to sign arbitrary transactions, although it will work for all transaction types
2021-11-05 14:37:47 +00:00
CalDescent
09a7fcaba4 Added MissingDataException
This is generated whenever a data resource cannot be built because it is missing data for at least one layer. Using a custom exception type here enables a few new features:

1. A single build process is now able to request missing data from all the layers that need it. Previously it would only request from the first missing layer and would then give up. This resulted in the user/application having to issue the build command multiple times rather than just once, until all layers had been requested.

2. GET /arbitrary/{service}/{name} will now block the response and retry in the background until the data arrives. This allows it to be used synchronously. Note: we'll need to add a timeout.

3. Loading a website via GET /site/{name} will avoid adding to the failed builds queue when a MissingDataException is thrown, which allows it to be quickly retried. The interface already auto refreshes, allowing the site to load as soon as it's available.
2021-11-04 09:09:54 +00:00
CalDescent
ce15784851 Return a detailed error message in GET /arbitrary/{service}/{name} 2021-11-03 21:55:55 +00:00
CalDescent
b861b2dffb Fixed loading bug when a transaction's data size is smaller than the chunk size. 2021-11-03 21:54:57 +00:00
CalDescent
e50fd786da Don't respond with a file list for a transaction that is outside of our storage policy, even if we do have a copy of the file at the time of the request. 2021-11-03 21:44:31 +00:00
CalDescent
5e82de667e Don't broadcast that we are holding files for a transaction unless it's within the scope of our storage policy.
We may need to temporarily hold files for the purpose of viewing, but restrictions need to be in place to avoid these being served to peers of stored for longer than they are needed.
2021-11-03 21:40:15 +00:00
CalDescent
d7ddcda9da Refactor to simplify some duplicated code 2021-11-03 21:39:02 +00:00
CalDescent
6d031130b9 Invalidate the cache in ArbitraryTransaction.onImportAsUnconfirmed() if we have the local data
This ensures that the local copy of a resource updates as soon as the transaction is broadcast.
2021-11-03 21:33:29 +00:00
CalDescent
a61b0685f0 Arbitrary data manager now only prefetches data according to the storage policy.
- If storage policy includes "FOLLOWING", only process transactions relating to the followed names.
- If storage policy is "ALL", process all transactions.
- If storage policy is "NONE" or "VIEWED", don't process or prefetch any data.
2021-11-03 20:24:38 +00:00
CalDescent
abfeafc823 Refactored to move all build-related code to a new ArbitraryDataBuildManager class
This is will be used to coordinate all build processes and threads. This way it keeps it separate from the ArbitraryDataManager class, which was getting a bit cluttered.
2021-11-03 19:56:52 +00:00
CalDescent
3a51be3430 ArbitraryDataBuildManager renamed to ArbitraryDataBuilderThread, as we will likely want to run multiple instances of this when we scale up. 2021-11-03 19:38:17 +00:00
CalDescent
ede4802ceb Converted ArbitraryTestTransaction to version 5
This fixes a failed serialization test when Transaction.getVersionByTimestamp() returns 5
2021-11-03 19:19:27 +00:00
CalDescent
fe79119809 Added PresenceTestTransaction, to allow SerializationTests.testTransactions() to be unblocked 2021-11-03 19:18:26 +00:00
CalDescent
6f07dc7852 Always overwrite existing data when building via the queue.
This fixes a significant bug that was interfering with updates.
2021-11-02 19:34:25 +00:00
CalDescent
16bcba6e2e When accessing a website or other data resource, request the chunks if we don't already have them.
This causes the build to fail on the first pass due to missing chunks, however it now fails with a message indicating that it should be retried shortly. The website loader is already set up in such a way that it will be automatically retried, during which time the loading screen is shown.

Also added code to remove the resource from the "failed builds list" once the chunks arrive, so that it is able to be rebuilt sooner than the FAILURE_TIMEOUT (currently 5 minutes).
2021-11-02 19:32:48 +00:00
CalDescent
1002acb021 Fixed error in log entry. 2021-11-02 09:15:55 +00:00
CalDescent
b771544c5d Added test to check website/data updates. 2021-11-02 09:09:54 +00:00
CalDescent
8c7f09c454 Fixed yet another case sensitivity issue. 2021-11-01 20:02:38 +00:00
CalDescent
618cffefb1 Made names case insensitive when using them as a search filter. 2021-11-01 19:44:43 +00:00
CalDescent
8fd37e857e Fixed case sensitivity issue when building from past transactions. 2021-11-01 19:41:50 +00:00
CalDescent
8218bfd24b Initial implementation of storage policies.
- Don't attempt to fetch data for transactions which fall outside of the storage policy
- Delete files relating to transactions that are no longer within the scope of the storage policy

Note: some additional work needs to be done to ensure that viewed files are deleted when using a storage policy that excludes "VIEWED" content.
2021-10-31 22:38:30 +00:00
CalDescent
cbb2dbffb9 The /arbitrary/search API endpoint now uses a string instead of an int for the "service", and shows a dropdown of possible values in the API documentation page. 2021-10-31 21:22:51 +00:00
CalDescent
528a838643 Fixed bug in previous commit 2021-10-31 21:20:59 +00:00
CalDescent
cbed6418e7 Added ability to filter arbitrary transactions by name when searching. 2021-10-31 21:07:14 +00:00
CalDescent
4882cc92a8 StoragePolicy enum moved to new ArbitraryDataStorageManager class 2021-10-31 20:32:32 +00:00
CalDescent
28fb11068e Added "storagePolicy" setting, including startup validation.
Possible values are:

FOLLOWED_AND_VIEWED (default)
FOLLOWED
VIEWED
ALL
NONE

Nothing is affected by this setting yet, but this will be added shortly.
2021-10-31 19:15:26 +00:00
CalDescent
90465149e6 Added API endpoints to allow registered names to be followed/unfollowed.
These will ultimately be used by the UI to control which data the node should host.
2021-10-31 18:42:18 +00:00
CalDescent
c6d868d981 "Lists" feature is now generic.
This means that no additional structural code is required to add new lists. The only non-generic aspect are the API endpoints - it's best to keep these specific until we have a need for user-created lists.
2021-10-31 18:39:58 +00:00
CalDescent
bada4fd140 Use a buffered digest where possible instead of reading the whole file contents into memory. 2021-10-30 18:21:06 +01:00
CalDescent
60f96d15bd When specifying a domain without a subdomain, add a www. version automatically.
Longer term we will probably need a 301 redirect in these cases for SEO purposes, but this is a nice convenience feature for now.
2021-10-30 17:47:57 +01:00
CalDescent
0328007345 Domain mapping now uses registered name instead of transaction signature.
There's no real need to maintain support for signature mapping anymore. Using this new method means that the latest version of a site is always served via the traditional domain name, whereas using transaction signatures caused older versions to be shown.

Example settings.json configuration:

"domainMapServiceEnabled": true,
"domainMapServicePort": 80,
"domainMap": [
  {
    "domain": "webdemo.qortal.uk",
    "name": "QortalDemo"
  },
 {
    "domain": "www.reqorder.org",
    "name": "ReQorder"
  }
]
2021-10-30 17:29:23 +01:00
CalDescent
3ad0e92a0f Log empty responses in qdata 2021-10-29 18:58:59 +01:00
CalDescent
3934120541 Return the data directly in GET /arbitrary/{service}/{name}, instead of a path to the data 2021-10-29 18:52:05 +01:00
CalDescent
24ca126f5a Various qdata improvements 2021-10-29 17:58:47 +01:00
CalDescent
651ca71126 Added transaction validity log. 2021-10-29 17:50:03 +01:00
CalDescent
e7cb33d8e2 Synchronize peer data lookups.
Without this we could broadcast the same data multiple times, due to more than one thread processing the same message from different peers.
2021-10-29 17:46:58 +01:00
CalDescent
c63d238316 Log the error response in the qdata utility if broadcasting a transaction fails. 2021-10-29 17:26:40 +01:00
CalDescent
dcdc48d917 Private key moved from command line argument to QORTAL_PRIVKEY env variable, to improve security. 2021-10-29 17:24:22 +01:00
CalDescent
f4c1671079 Removed the need to include public key in recently added API endpoints, as it can be derived from the registered name's owner. 2021-10-29 17:08:26 +01:00
CalDescent
7aa2fbee1c Added "qdata" command line tool to host and retrieve data 2021-10-29 16:51:41 +01:00
CalDescent
f1939fdc2b Added generic PUT, PATCH, and GET /arbitrary API endpoints
These will likely avoid the need for some of the /site APIs.
2021-10-29 16:45:08 +01:00
CalDescent
c9356d0ff5 Re-broadcast the arbitrary signatures message if it contains new data, so that the message finds its way to all online peers. 2021-10-29 16:16:58 +01:00
CalDescent
6b5d938a40 Added saveChanges() missing from previous commit, and a discardChanges() just in case. 2021-10-29 16:16:05 +01:00
CalDescent
d82da160f3 Added DHT-style lookup table to track file locations
This maps ARBITRARY transactions to peer addresses, but also includes additional metadata/stats to track the success rate and reachability.

Once a node receives files for a transaction, it broadcasts this info to its peers so they can update their records.

TLDR: this allows us to locate peers that are hosting a copy of the file we need.
2021-10-29 13:35:17 +01:00
CalDescent
54c8aac20d Commented out unused forwarding code 2021-10-29 13:27:37 +01:00
CalDescent
05be5c1199 Version set to 2.0.0, given that this will ultimately be merged into the main qortal core. It makes sense to keep the version numbers in sync. 2021-10-25 19:06:48 +01:00
CalDescent
f19a65148a Revert "Default minPeerVersion set to 0.1.0. TODO: revert this if ever merged into the main repo."
This reverts commit 9b4d832d17a8b96f395daa6cc98944a68b54f4fa.
2021-10-25 19:04:31 +01:00
CalDescent
a55fc4fff9 When validating an ARBITRARY transaction, ensure that the supplied name exists and is registered to the account that is signing the transaction.
This ensures that only the owner of a name is able to update data associated with that name.

Note that this doesn't take into account the ability for group members to update a resource, so this will need modifying when that feature is ultimately introduced (likely after v3.0)
2021-10-25 18:58:33 +01:00
CalDescent
35a7a70b93 Merge remote-tracking branch 'qortal/master' 2021-10-25 18:26:06 +01:00
CalDescent
1b846be5fc Fixed bug in arbitrary transaction builder 2021-10-24 22:43:58 +01:00
CalDescent
707eb58068 Added testPatchBeforePut() unit test 2021-10-24 22:43:28 +01:00
CalDescent
8630f3be96 Added first end-to-end test of data storage 2021-10-24 19:08:09 +01:00
CalDescent
c90aeba286 Return ArbitraryTransactionData instead of a byte array, as it is more useful if the bytes are transformed separately. 2021-10-24 19:06:30 +01:00
CalDescent
5055cfc6cb Removed unused imports 2021-10-24 17:43:28 +01:00
CalDescent
c222c4eb29 Updated expected hash of demo data as it has been updated. 2021-10-24 17:41:51 +01:00
CalDescent
6c01955561 Refactor to move arbitrary transaction building to its own class. 2021-10-24 17:41:28 +01:00
CalDescent
305e0f1772 Disable validation of previous hash unless validateAllDataLayers is true
We may not need to validate this at all now that we have the ability to validate the current layer, but I'll leave it as it could be useful for debugging. It is disabled by default so not an issue.
2021-10-24 15:35:51 +01:00
CalDescent
52a94e3256 Added "validateAllDataLayers" setting (default false)
When true, the hashes of every layer are validated when building a data resource. When false, only the final layer's hash is validated.
2021-10-24 14:37:29 +01:00
CalDescent
a418fb18b6 Hash the current state when creating a patch
This is included in the patch metadata and then validated every time it is rebuilt.
2021-10-24 13:00:21 +01:00
CalDescent
9cd579d3db Another typo 2021-10-24 12:20:49 +01:00
CalDescent
e1a6ba7377 Fixed incorrect comment. 2021-10-24 12:03:22 +01:00
CalDescent
04aabe0921 Include the original file instead of a patch if the patch is larger than the original file.
This saves processing and disk space, as there is no point in applying a patch when the original file is smaller and can be included in its entirety.
2021-10-24 12:02:09 +01:00
CalDescent
8dd4d71d75 Significant rework of patches
- The "diff type" is now specified per file, allowing for different diff methods in each modified file.
- Patches will only be created when both the before and after files are less than 100kiB in size.
- Patches are validated after creation, and if invalid it will fall back to including the entire file.

This has identified a bug where patching fails for files without trailing newline characters, which still needs to be fixed. Until then, it will fall back to including the entire file in these cases.
2021-10-24 10:47:47 +01:00
CalDescent
12b3267d5c Added arbitrary data merge tests. 2021-10-22 11:49:15 +01:00
CalDescent
d6d564c027 Fixed refresh interval of loading screen 2021-10-17 16:55:32 +01:00
CalDescent
f0e13fa492 Arbitrary transaction names are now case insensitive 2021-10-15 13:58:27 +01:00
CalDescent
c8d5ac9248 Fixed bug in ArbitraryTransactionTransformer.getDataLength() when missing a name. 2021-10-15 11:32:08 +01:00
CalDescent
aa4f77d4de Fixed merge issues relating to database updates. Existing data nodes will need to delete their db folder and resync. 2021-10-15 09:13:15 +01:00
CalDescent
f3ef112297 Merge remote-tracking branch 'qortal/master'
# Conflicts:
#	.gitignore
#	pom.xml
#	src/main/java/org/qortal/controller/Controller.java
#	src/main/java/org/qortal/gui/SysTray.java
#	src/main/java/org/qortal/settings/Settings.java
#	src/main/resources/i18n/ApiError_en.properties
#	src/test/java/org/qortal/test/CryptoTests.java
#	src/test/resources/test-settings-v2.json
2021-10-15 09:03:28 +01:00
CalDescent
9b4d832d17 Default minPeerVersion set to 0.1.0. TODO: revert this if ever merged into the main repo. 2021-09-01 09:11:50 +01:00
CalDescent
52ab19dec6 Added method and name to the /site/upload endpoint params. 2021-09-01 09:11:03 +01:00
CalDescent
7397b9fa87 Added more detail to exception message. 2021-08-21 08:35:12 +01:00
CalDescent
5bed5fb8fd Removed unnecessary code in ArbitraryResource.uploadFileAtPath()
This is now handled by ArbitraryDataWriter instead
2021-08-21 08:34:46 +01:00
CalDescent
fd795b4361 Don't attempt to cleanup the filesystem if a build is in progress.
This isn't essential but it helps to reduce unnecessary load and processing which would be better spent on building.
2021-08-20 20:19:51 +01:00
CalDescent
b2c0915a71 Removed accidentally duplicated code
This was causing two instances of the build manager to run.
2021-08-20 20:17:02 +01:00
CalDescent
095083bcfb Use lowercase directory names for consistency 2021-08-20 19:54:09 +01:00
CalDescent
4ba72f7eeb Regularly clean up old and unused files/folders in the temp directory
Also added code to purge built resource caches, but it is currently disabled. This will become more useful once we implement local storage limits.
2021-08-20 19:27:42 +01:00
CalDescent
6cb39795a9 Removed requirement to have connected peers in order to cleanup directories. 2021-08-20 17:00:53 +01:00
CalDescent
00ba16f536 Fixed incorrect ArbitraryTransactionTransformer layout. 2021-08-20 16:59:05 +01:00
CalDescent
988a839623 Improved response value of ArbitraryDataFile.deleteAllChunks() as it was inaccurate. 2021-08-20 16:58:43 +01:00
CalDescent
8fa61e628c Delete files related to transactions that have a more recent PUT
A PUT creates a new base layer meaning anything before that point is no longer needed. These files are now deleted automatically by the cleanup manager. This involved relocating a lot of the cleanup manager methods into a shared utility, so that they could be used by the arbitrary data manager. Without this, they would be fetched from the network again as soon as they were deleted.
2021-08-20 16:56:49 +01:00
CalDescent
8f3620e07b Fixed bug introduced in commit 51b1256 2021-08-20 13:44:30 +01:00
CalDescent
190f70f332 Removed unused, buggy code in HSQLDBArbitraryRepository.save()
It's safer to throw an exception and point the user towards ArbitraryDataWriter, rather than maintaining unused code.
2021-08-20 13:01:55 +01:00
CalDescent
6730683919 Added arbitrary data cleanup manager
This deletes redundant copies of data, and also converts complete files to chunks where needed. The idea being that nodes only hold chunks, since they currently are much more likely to serve a chunk to another peer than they are to serve a complete file.

It doesn't yet cleanup files that are unassociated with transactions, nor does it delete anything from the _temp folder.
2021-08-20 12:55:42 +01:00
CalDescent
51b12567e8 Switched ArbitraryDataFile.filePath from String to Path 2021-08-20 09:17:36 +01:00
CalDescent
d01cdeded8 Added pagination to ArbitraryDataManager
This improves scalability but isn't sufficient for a long term solution. TODO: It probably makes sense to add an additional query for recent transactions only, so that they are fetched quickly.
2021-08-20 07:58:55 +01:00
CalDescent
d22a03f1a5 Fixed misleading exception string. 2021-08-20 07:51:58 +01:00
CalDescent
ab0aeec434 Default method of serving websites switched from signature to name.
Before:
GET /site/:signature
GET /site/name/:name

After:
GET /site/signature/:signature
GET /site/:name
2021-08-20 07:51:27 +01:00
CalDescent
47ff51ce4e Use a random last reference on the very first transaction for an account
This is needed because we want to allow brand new accounts to publish data without a fee. A similar approach to CrossChainResource.buildAtMessage(). We already require PoW on all arbitrary transactions, so no additional logic beyond this should be needed.
2021-08-20 07:49:05 +01:00
CalDescent
1d62ef357d Check for null last references at the beginning of the process when creating an arbitrary tx. 2021-08-19 09:10:13 +01:00
CalDescent
59ef66f46d Handle shutdowns when zipping a large number of files. 2021-08-19 08:48:39 +01:00
CalDescent
77479215a6 Throw an exception if a patch file doesn't exist. 2021-08-18 20:32:47 +01:00
CalDescent
11da1f72b1 Added convenience method to make the code more readable. 2021-08-18 20:30:00 +01:00
CalDescent
6375b9d14d Fixed bug in getLatestTransaction() filtering. 2021-08-18 20:27:18 +01:00
CalDescent
42bc12f56d Call cleanupQueues() from the Controller 2021-08-18 16:25:47 +01:00
CalDescent
8515112811 Fixed unused variable issue 2021-08-18 16:24:37 +01:00
CalDescent
bedb87674b FAILURE_TIMEOUT set to 5 minutes 2021-08-18 16:24:13 +01:00
CalDescent
029c038a49 Catch runtime exceptions (e.g. IllegalStateException) when using the arbitrary data reader/writer. 2021-08-18 16:23:46 +01:00
CalDescent
95e905a5ae If a build fails, prevent any rebuilds for 5 minutes.
This prevents a resource with build problems from getting into a loop due to the browser requesting a rebuild as soon as it fails.
2021-08-18 16:04:57 +01:00
CalDescent
8a8ec32f2c Added java-diff-utils to pom.xml
This was accidentally missing from commit cb6fc46.
2021-08-18 09:59:56 +01:00
CalDescent
cd958398af Exclude all data* paths from gitignore. 2021-08-18 09:59:12 +01:00
CalDescent
2eedafd506 Log the error if a build fails. 2021-08-18 09:58:59 +01:00
CalDescent
9a88c0d579 Apply the unified-diff patch when combining layers. 2021-08-18 09:58:30 +01:00
CalDescent
cb6fc466d1 Create a "unified-diff" patch using java-diff-utils instead of including the entire modified file. I still need to test how well this works with binary files. 2021-08-18 09:57:44 +01:00
CalDescent
a6154cbb43 Don't allow a new layer to be created if it matches the existing state. 2021-08-18 07:51:50 +01:00
CalDescent
9c20967d24 Catch NPE seen a couple of times in Systray.setTrayIcon() 2021-08-18 07:51:26 +01:00
CalDescent
79bbadad2f Initial implementation of arbitrary data build queue
This adds the loadAsynchronously() method to ArbitraryDataReader, in addition to the existing loadSynchronously() method.

When requesting a website in a browser, previously the building of the resource's layers would be done synchronously in the API handler. This understandably caused many issues, so the building is now done asynchronously by a dedicated thread. A loading screen is shown in its place which auto refreshes every second until the build has completed.
2021-08-18 07:50:45 +01:00
CalDescent
c3b44cee94 Fixed bugs when computing directory digest. The order was being lost when adding to the second List, which was producing different hashes on different systems. It also doesn't make sense to convert paths to lowercase, given that we want our validation to be case sensitive. 2021-08-18 07:38:08 +01:00
CalDescent
1968496ce1 Invalidate the cache if hash validation fails, so that it can be rebuilt next time. 2021-08-17 09:30:27 +01:00
CalDescent
e1feb46de9 Put test data in a separate folder to real data. 2021-08-17 09:08:03 +01:00
CalDescent
f51a082049 Validate the previous state's hash each time a new layer is applied.
It's possible that this concept will struggle in the real world if operating systems, virus scanners, etc start interfering with our file stucture. Right now it is using a zero tolerance approach when checking the validity of each layer. We may choose to loosen this slightly if we encounter problems, e.g. by excluding hidden files. But for now it is best to be as strict as possible.
2021-08-17 09:07:46 +01:00
CalDescent
0f1927b4b1 Take a hash of the previous state's directory structure and file contents, and then include that hash in the patch metadata (when creating a new patch). This allows the integrity of the layers to be validated as each one is applied. 2021-08-17 08:43:08 +01:00
CalDescent
9baccc0784 Improved HTTP response generation when serving websites. 2021-08-16 22:40:55 +01:00
CalDescent
95c9cc7f99 Added ArbitraryDataCache
This decides whether to build a new state or use an existing cached state when serving a data resource. It will cache a built resource until a new transaction (i.e. layer) arrives. This drastically reduces load, and still allows for almost instant propagation of new layers.
2021-08-16 22:40:05 +01:00
CalDescent
0ed8e04233 Arbitrary data metadata classes moved to their own package. 2021-08-15 21:54:21 +01:00
CalDescent
b46c328811 Fixed bug in FilesystemUtils.copyAndReplaceDirectory() 2021-08-15 21:50:19 +01:00
CalDescent
c7d88ed95b Added "cache" file to the .qortal metadata folder.
This is used to store the transaction signature and build timestamp for each built data resource. It involved a refactor of the ArbitraryDataMetadata class to introduce a subclass for each file ("patch" and "cache"). This allows more files to be easily added later.
2021-08-15 21:49:45 +01:00
CalDescent
94da1a30dc When merging two states, validate that the transaction signature we are using for the "before" layer matches the "previous transaction signature" that is baked into the "after" layer.
This defends against a missing or out-of-order transaction. If this ever fails validation, we may need to rethink the way we are requesting transactions. But in theory this shouldn't happen, given that the "last reference" field of a transaction ensures that out-of-order transactions are invalid already.
2021-08-15 20:27:16 +01:00
CalDescent
219a5db60c Fixed circular bug in ArbitraryDataMetadata.getPreviousSignature() 2021-08-15 20:21:47 +01:00
CalDescent
a5cfedcae9 Exclude the .qortal metadata directory in all diffs. Also improved logging. 2021-08-15 20:02:14 +01:00
CalDescent
9b8a632f37 Include the .qortal folder in the merge output, since it will be needed for validation. We may choose to exclude it from the final output path, but for now it is left there to make debugging easier. 2021-08-15 19:55:56 +01:00
CalDescent
be0426d9a2 Improved logging 2021-08-15 19:19:55 +01:00
CalDescent
8fac0a02e5 Use the /.qortal/patch file to apply differences when merging together two layers, rather than walking through the file tree like we did before. 2021-08-15 18:49:30 +01:00
CalDescent
fa696a2901 Log exceptions when publishing data updates. 2021-08-15 18:37:27 +01:00
CalDescent
f5615b1c54 Don't exclude hidden files in the zips, as the .qortal folder needs to be included. 2021-08-15 17:45:37 +01:00
CalDescent
9850c294d1 Fixed various issues relating to syncing data for transactions without any chunks. 2021-08-15 11:28:26 +01:00
CalDescent
8929f32068 No longer creating the ".removed" files. 2021-08-15 10:43:00 +01:00
CalDescent
3019bb5c97 Enforce minBlockchainPeers in ArbitraryDataManager, as there is no point in trying to request data files when we don't have the minimum number of peers. 2021-08-15 10:39:00 +01:00
CalDescent
95044d27ce Fixed NPE caused by having an arbitrary transaction with no chunks (which is expected if the total data size is less than the chunk size). 2021-08-15 10:32:37 +01:00
CalDescent
16ac92b2ef Write patch metadata to a file inside a hidden ".qortal" folder which is included with each patch. This can be used in place of the existing ".removed" placeholder files to track removals. 2021-08-15 10:13:32 +01:00
CalDescent
f095964f7b Fixed edge case. 2021-08-14 18:28:22 +01:00
CalDescent
cfba793fcf Fixed bugs in ArbitraryDataFile, introduced when switching to the new temporary path. 2021-08-14 18:10:59 +01:00
CalDescent
c50a11e58a Cleanup intermediate paths in ArbitraryDataWriter. 2021-08-14 17:32:25 +01:00
CalDescent
a7282a5794 Renamed encrypted and zipped files. 2021-08-14 17:31:58 +01:00
CalDescent
efaf313422 When creating an ArbitraryDataFile from a path, make sure to move the file to the correct location within the data directory.
This bug was introduced now that the temp directory is contained within the data directory. Without this, it would leave it in the temp folder and then fail at a later stage.
2021-08-14 16:58:10 +01:00
CalDescent
d9f5753f58 Ensure parent directories exist when creating a diff. 2021-08-14 16:55:19 +01:00
CalDescent
179318f1d8 Ensure a patch exists after creating it. 2021-08-14 16:54:35 +01:00
CalDescent
dd33d24346 Fixed bug with identifier in ArbitraryDataWriter 2021-08-14 16:53:57 +01:00
CalDescent
fa684eabab Removed ArbitraryDataPatches class
This was essentially just a wrapper for a single method.
2021-08-14 15:51:16 +01:00
CalDescent
4b5bd6eed7 Cleanup temporary files used when building data directories. Also added a safety net to only delete files that are without our data or temp directories. 2021-08-14 15:43:08 +01:00
CalDescent
63f5946527 Switched all system-generated temp paths to a user specified "tempDataPath".
This ensures that the temporary files are being kept with the rest of the data, rather than somewhere inappropriate such as on flash storage. It also allows the user to locate them somewhere else, such as on a dedicated drive.
2021-08-14 14:30:06 +01:00
CalDescent
8dac3ebf96 More miscellaneous refactoring 2021-08-14 14:05:33 +01:00
CalDescent
09e783fbf6 Renamed a few variables and methods that weren't caught in previous refactors. 2021-08-14 14:01:30 +01:00
CalDescent
b18b686545 Renamed DATA_FILE to ARBITRARY_DATA_FILE and GET_DATA_FILE to GET_ARBITRARY_DATA_FILE.
This will break existing data nodes, but this is okay, as nothing is in production yet.
2021-08-14 13:57:44 +01:00
CalDescent
e99ea41117 Renamed DataFileMessage to ArbitraryDataFileMessage and GetDataFileMessage to GetArbitraryDataFileMessage 2021-08-14 13:52:41 +01:00
CalDescent
172a37ec8c Renamed org.qortal.storage package to org.qortal.arbitrary 2021-08-14 13:49:19 +01:00
CalDescent
da6b341b63 Renamed DataFile to ArbitraryDataFile, and DataFileChunk to ArbitraryDataFileChunk 2021-08-14 13:25:45 +01:00
CalDescent
16d93b1775 Renamed newly added classes to ArbitraryData*.java instead of DataFile*.java 2021-08-14 13:18:02 +01:00
CalDescent
e15cf063c6 Initial implementation of data patches/updates
This adds support for the PATCH method in addition to the existing PUT method.

Currently, a patch includes only files that have been added or modified, as well as placeholder files to indicate those that have been removed.

This is not production ready, as I am hoping to create patches on a more granular level - i.e. just the modified bytes of each file. It would also make sense to track deletions using a metadata/manifest file in a hidden folder.

It also adds early support of accessing files using a name rather than a signature or hash.
2021-08-14 13:11:36 +01:00
CalDescent
5ac9e3e47a Fixed recently introduced bugs in arbitrary transaction transformation. 2021-08-13 19:14:44 +01:00
CalDescent
743a61bf49 Fixed bug which overwrote the data file, causing it to be invalid. 2021-08-13 06:55:44 +01:00
CalDescent
c790ea07dd Started abstracting the file processing code away from the API handlers, and making it more modular. 2021-08-12 20:53:42 +01:00
CalDescent
e259a09b89 Fixed merge issues. 2021-08-01 09:54:15 +01:00
CalDescent
6472d8438a Merge remote-tracking branch 'qortal/master'
# Conflicts:
#	pom.xml
#	src/main/java/org/qortal/api/ApiError.java
#	src/main/java/org/qortal/settings/Settings.java
#	src/main/resources/i18n/ApiError_en.properties
2021-07-31 22:57:07 +01:00
CalDescent
dc8a402a4a Revert "Removed all cross-chain code. It's not needed for data nodes."
This reverts commit cc59510cd05ffb3f4845ee7d5389ac197a8fd0fc.

# Conflicts:
#	src/main/java/org/qortal/controller/Controller.java
2021-07-31 22:55:27 +01:00
CalDescent
f72374488e Revert "Removed block 212937"
This reverts commit b917da765c9406adacbb5abd261c3b6ed236ec74.
2021-07-31 22:54:19 +01:00
CalDescent
7a77b12834 Repalce &amp; with & in HTML documents.
This is not a long term solution, but it's a quick fix to solve URL problems when converting to a static site via httrack.
2021-07-17 19:18:22 +01:00
CalDescent
fe387931a4 Increased buffer when serving assets from 1kiB to 10kiB
This speeds up image loading. Ultimately we will stop serving these via the application memory altogether.
2021-07-17 15:48:27 +01:00
CalDescent
dc83e32173 Fixed site preview functionality. 2021-07-17 15:23:24 +01:00
CalDescent
f5b29bad33 Encrypt websites with AES.
This ensures that nodes are storing unreadable files, outside of the context of Qortal. For public data, the decryption keys themselves are on-chain, included in the "secret" field of arbitrary transactions. When we introduce the concept of private data, we can simply exclude the secret key from the transaction so that only the owner can decrypt it.

When encrypting the file, I have added the 16 byte initialization vector as a prefix to the cyphertext, and it is then automatically extracted back out when decrypting. This gives us the option to encrypt more than one file with the same key, if we ever need it. Right now, we are using a unique key per file, so it's not actually needed, but it's good to have support.
2021-07-17 14:51:39 +01:00
CalDescent
f599aa4852 Modified serializeSizedString() and deserializeSizedString() to cope with null strings.
This affects various other parts of the system, not just arbitrary transactions.
2021-07-17 14:43:02 +01:00
CalDescent
6f05de2fcc Fixed newly introduced issues with arbitrary transaction transformation 2021-07-17 14:42:00 +01:00
CalDescent
02016c77f1 Fixed issue in HSQLDBSaver, introduced in recent commit. 2021-07-17 14:40:29 +01:00
CalDescent
93eede7c6b Added missing break statement which caused a database version mismatch. 2021-07-15 19:56:09 +01:00
CalDescent
944e396823 Added AES utility class from baeldung and updated copyright notice for ZipUtils which was based on code from the same author. This code still needs reviewing and modifying but it's a good starting point for AES encryption and decryption. 2021-07-15 19:50:42 +01:00
CalDescent
8a654834ac Small reorganization. 2021-07-15 19:47:53 +01:00
CalDescent
bb76fa80cd Another significant upgrade of arbitrary transactions
Adds "name", "method", "secret", and "compression" properties. These are the foundations needed in order to handle updates, encryption, and name registration. Compression has been added so that we have the option of switching to different algorithms whilst maintaining support for existing transactions.
2021-07-15 09:27:49 +01:00
CalDescent
53f44a4029 Added support for subdirectories in the HTML parser. 2021-07-14 18:03:51 +01:00
CalDescent
182dcc7e5f MAX_FILE_SIZE reduced to 500MiB to match the difficulty calculation. 2021-07-14 18:03:35 +01:00
CalDescent
2d272e0207 Added some service constants.
These combine some Qora services (SERVICE_NAME_STORAGE, SERVICE_BLOG_POST, and SERVICE_BLOG_COMMENT) with existing Qortal services (SERVICE_AUTO_UPDATE), and some new additions (SERVICE_ARBITRARY_DATA, SERVICE_WEBSITE, and SERVICE_GIT_REPOSITORY)
2021-07-14 17:50:55 +01:00
CalDescent
9384a50879 Derive PoW difficulty from the file size. Exact values TBC. 2021-07-13 22:18:21 +01:00
CalDescent
00d4f35f2c Track the request time in arbitraryDataFileRequests and automatically remove those that have timed out. 2021-07-11 10:28:14 +01:00
CalDescent
483557163e Refactor: moved arbitrary data code from Controller to ArbitraryDataManager 2021-07-11 10:17:38 +01:00
CalDescent
2679252b04 Rework of arbitrary data requests
Previously we would ask all connected peers for the file itself, but this caused the network to be swamped when multiple peers responded with the same file.

This new approach instead asks all connected peers to send back a list of hashes for all files they have relating to a transaction signature. The requesting node then uses these lists to make separate requests for each missing file.
2021-07-11 10:00:09 +01:00
CalDescent
5a95c827b4 When serving a website, delete the unzipped directory if the index file is not found.
This is a quick solution to rebuild directory structures with missing files. This whole area of the code needs some reworking, as serving the site from a temporary folder is not a robust long term solution.
2021-07-10 15:51:46 +01:00
CalDescent
f938d8c878 More refactoring 2021-07-10 14:47:51 +01:00
CalDescent
bfc0122c1b Added warning to README 2021-07-10 13:53:07 +01:00
CalDescent
cdc5348a06 Added "domain map" server
Domain names can be mapped to arbitrary transaction signatures via the node's settings, and then served over port 80 or 443. This allows Qortal hosted sites to be accessible via a traditional domain name.

Example configuration to map two domains:

"domainMapServiceEnabled": true,
"domainMapServicePort": 80,
"domainMap": [
  {
    "domain": "example.com",
    "signature": "tEsw4kUn4ZJfPha7CotUL6BHkFPs79BwKXdY6yrf28YTpDn4KSY6ZKX3nwZCkqDF9RyXbgaVnB1rTEExY3h9CQA"
  },
  {
    "domain": "demo.qortal.org",
    "signature": "ZdBWWPMhR7AZwSu5xZm89mQEacekqkNfAimSCqFP6rQGKaGnXR9G4SWYpY5awFGfhmNBWzvRnXkWZKCsj6EMgc8"
  }
]

Each domain needs to be pointed to the Qortal data node via an A record or CNAME. You can add redundant nodes by adding multiple A records for the same domain (this is known as DNS Failover).

Note that running a webserver on port 80 (or anything less than 1024) requires running the data node as root. There are workarounds to this, such as disabling privileged ports, or using a reverse proxy. I will investigate this more as time goes on, but this is okay for a proof of concept.
2021-07-06 18:50:40 +01:00
CalDescent
e64a3978e6 Moved HTML parsing to new class. 2021-07-06 08:23:28 +01:00
CalDescent
f2feb12708 /site API endpoint now tales a signature rather than a file hash
This allows it to verify that the data in a transaction, after which it will then build the complete file from its chunks if needed.
2021-07-05 09:07:06 +01:00
CalDescent
5319c5f832 Reworked existing unused ArbitraryDataManager.
It's now capable of syncing chunks as well as complete files. This isn't production ready as it currently requests/receives the same file from multiple peers at once, which slows down the sync and wastes lots of bandwidth. Ideally we would find an appropriate peer first and then sync the file from them.
2021-07-05 09:05:45 +01:00
CalDescent
7531fe14fe Fixed major performance issue in DataFile.toString() 2021-07-05 08:23:29 +01:00
CalDescent
0086c6373b Significant refactor of DataFile and DataFileChunk
This introduces the hash58 property, which stores the base58 hash of the file passed in at initialization. It leaves digest() and digest58() for when we need to compute a new hash from the file itself.
2021-07-05 07:26:20 +01:00
CalDescent
10dc19652e Use "qortaldata-" version prefix 2021-07-04 16:52:20 +01:00
CalDescent
2f2c4964c5 Transaction version temporarily bumped to 5. TODO: We will need to set a hard fork timestamp if this is ever merged back into the main Qortal core repo. 2021-07-04 16:43:12 +01:00
CalDescent
cb4203b6db Use public key as parameter instead of address, since we can obtain the address from the public key in all cases. 2021-07-04 14:53:54 +01:00
CalDescent
bb5b62466e Fixed bug introduced in recent commit. 2021-07-04 13:41:37 +01:00
CalDescent
6407b5452b Delete our copies of data if any exception is thrown. 2021-07-04 13:39:00 +01:00
CalDescent
a742fecf9c API refactors to avoid generic unhandled states. 2021-07-04 13:38:20 +01:00
CalDescent
60415b9222 /arbitrary/upload/creator/{address} now returns an unsigned ARBITRARY transaction, currently with pre-computed nonce (same as the /site equivalent) 2021-07-04 10:25:15 +01:00
CalDescent
ffb39ef074 /data API endpoints moved to /arbitrary 2021-07-04 09:49:01 +01:00
CalDescent
d73f5ed2b5 /site/upload/creator/{address} now returns an unsigned ARBITRARY transaction, currently with pre-computed nonce 2021-07-04 09:45:52 +01:00
CalDescent
7af973b60d Removed TODO comments that are now done. 2021-07-04 09:41:53 +01:00
CalDescent
49eddc9da5 Allow zero fee transactions if the fee is zero in blockchain.json
Until now it wasn't possible to set up a chain with zero transaction fees due to a hardcoded zero check in Payment.isValid(), and a divide by zero error in Transaction.hasMinimumFeePerByte()
2021-07-04 09:31:51 +01:00
CalDescent
4b1de108d1 Fixed bug in expected chunk count. 2021-07-03 18:42:42 +01:00
CalDescent
e46c735efa Fixed recently introduced bugs with file management. 2021-07-03 18:05:17 +01:00
CalDescent
56da7deb4c DataFile updates to simplify integration with arbitrary transactions. 2021-07-03 17:41:52 +01:00
CalDescent
5f4649ee2b Major upgrade of arbitrary transactions
- Adds support for files up 500MiB per transaction (at 2MiB chunk sizes). Previously, the max data size was 4000 bytes.
- Adds a nonce, giving us the option to remove the transaction fees altogether on the data chain.

These features become enabled in version 5 of arbitrary transactions.
2021-07-03 17:40:02 +01:00
CalDescent
7cc2c4f621 Progress on website API endpoints. 2021-07-02 08:57:55 +01:00
CalDescent
cc449f9304 Added DataFile.chunkHashes() method which appends all hashes into a single byte array (32 bytes / 256 bits each). 2021-07-02 08:54:05 +01:00
CalDescent
28425efbe7 Added jsoup dependency - this was missing from an earlier commit. 2021-07-02 08:52:38 +01:00
CalDescent
8c3a22aa5c Improved link replacement criteria. 2021-07-01 11:21:40 +01:00
CalDescent
f3e5933599 Fixed naming error in joinFiles API. 2021-07-01 11:18:46 +01:00
CalDescent
aac4fe37e8 Fixed API response description. 2021-06-26 10:28:22 +01:00
CalDescent
ebfa941a4f Fixed some file separators. 2021-06-26 10:27:32 +01:00
CalDescent
47c70eea9e Use system temp directory instead of making a "temp" subfolder when zipping files. 2021-06-26 10:11:34 +01:00
CalDescent
fe7c40cb7c Reduced log spam. 2021-06-25 19:31:28 +01:00
CalDescent
8973626a4b Fixed issue with temp directory on Linux. 2021-06-25 19:31:13 +01:00
CalDescent
ace5d999e2 Log a comma separated list of hashes after splitting a file into chunks, so they can easily be requested from another node using the //data/files/frompeer/{peer} API endpoint. Again temporary until the sync happens automatically. 2021-06-25 19:30:45 +01:00
CalDescent
52829a244b More work on APIs to request files from peers. I won't spend too long making these perfect as they are mostly a temporary measure. 2021-06-25 19:28:39 +01:00
CalDescent
71c247fe56 Added POST /data/file/{hash}/build API, used to join multiple chunks together. 2021-06-25 19:28:01 +01:00
CalDescent
b34066f579 More work on HTML parsing.
The style tag parsing ideally needs rewriting using an actual CSS parser, but we can get away with this hacky approach in the short term.
2021-06-25 08:34:44 +01:00
CalDescent
b286c15c51 Optimized website serving, and added code to return the correct content types.
This is probably the most efficient way to process the data on the fly, but it's still not very scalable. A better approach would be to pre-process the HTML when building the file structure, and then serve them completely statically (i.e. using a standard webserver rather than via application memory). But it makes sense to keep it this way for development and maybe early beta testing.
2021-06-25 08:32:22 +01:00
CalDescent
ea5e2f5580 Added POST /site/preview API
This can be used to preview a site before signing a transaction and announcing it to the network. The response will need reworking to return JSON (along with most of the other new APIs)
2021-06-23 09:28:38 +01:00
CalDescent
b65c7a75fe Handle relative links when parsing HTML. 2021-06-23 09:26:41 +01:00
CalDescent
39f5dce51c Moved "directory" data uploads to new POST /site/upload API.
Directory uploads don't make much sense outside of website hosting, so it's best to make this API specific to that purpose.
2021-06-23 09:10:06 +01:00
CalDescent
f77ec1faf6 A very crude proof of concept which serves a website from a zipped (and in future, chunked) data blob. This forms the beginnings of the "website hosting" layer on top of the data storage. It needs a significant rework - most importantly so that we aren't serving every asset from memory, and also so that the correct content-type headers are returned, etc. 2021-06-23 08:43:21 +01:00
CalDescent
cd3a1e0159 Increased max file size to 1GiB. Will review this again later. 2021-06-23 08:33:53 +01:00
CalDescent
3f20fadb81 When zipping files, rename the outer folder to "data" instead of using the original folder name. This means that the data can be accessed deterministically without the need to first lookup the folder name. 2021-06-23 08:09:59 +01:00
CalDescent
1c6428dd3b Added equivalent split and join test but this time using a 5.5MiB source file. 2021-06-23 08:07:35 +01:00
CalDescent
aca620241a More work on data file split/join, and added a test. 2021-06-22 08:58:16 +01:00
CalDescent
808b36e088 Specify chunk size when splitting. 2021-06-22 07:44:34 +01:00
CalDescent
1613375cc0 Added more validation of files received in GET /data/file/frompeer 2021-06-21 19:03:34 +01:00
CalDescent
787ef957d2 Added support for uploading an entire directory via POST /data/upload/path
If a directory is specified instead of a file, the directory is automatically zipped before being split into chunks.
2021-06-21 19:02:49 +01:00
CalDescent
b915d0aed5 Only create the output file directories when we are actually writing a file there. This should prevent empty directories being created when initializing a nonexistent DataFile using a hash. 2021-06-21 08:40:52 +01:00
CalDescent
16dc5b5327 Include the data length in DataFileMessage, which is more similar to the approach used by ArbitraryDataMessage. These two message types are very similar, except arbitrary code currently has a requirement of one piece of data per signature, whereas DataFile code is independent and can support multiple files per transaction. Maybe the two can be combined at some point, but for now I'll keep them separate. 2021-06-20 19:49:10 +01:00
CalDescent
c2d0c63db0 Improved error logging. 2021-06-19 20:31:04 +01:00
CalDescent
f5c9807a48 Use contains() rather than equals() when matching a peer in /data/file/frompeer, so that the port can be optionally left out. 2021-06-19 20:29:05 +01:00
CalDescent
7e9b1d5e16 Rework of DataFile.base58Digest()
This fixes an NPE when trying to send a file that doesn't exist. It also removes the caching, which we can add again later if it turns out to be needed.
2021-06-19 20:25:25 +01:00
CalDescent
5070c4eea9 Better handling of data file responses in the /data/file/frompeer API endpoint. 2021-06-19 20:23:33 +01:00
CalDescent
33d9c51b6f Validate supplied base58 string in /data/file/frompeer API endpoint 2021-06-19 19:26:13 +01:00
CalDescent
d0f9d478c2 Fixed bug which prevented the DATA_FILE message ID from making it through to the reply queue. 2021-06-19 19:05:11 +01:00
CalDescent
f296ec46c8 Removed unused headers. 2021-06-19 17:32:55 +01:00
CalDescent
64d19e480b Chunk size set to 1MB for now, as it seems that our networking code has problems when transferring 2MB chunks. We can increase this later once that problem has been fixed. 2021-06-19 17:32:48 +01:00
CalDescent
8e35f131d5 Removed debugging log that wasn't intended to be committed. 2021-06-18 08:33:51 +01:00
CalDescent
aafb9d7e4f Include running total in "Sent X bytes" log entry. 2021-06-18 08:05:35 +01:00
CalDescent
652f30bdbd Added DATA_FILE and GET_DATA_FILE message types. 2021-06-18 08:02:57 +01:00
CalDescent
f4ba7b2a0c Added onNetworkGetDataFileMessage() handler 2021-06-18 08:02:13 +01:00
CalDescent
592490d709 Added GET /data/file/frompeer API endpoint
This requests a file from the supplied peer address, and stores a copy locally if successful. Still a work in progress.
2021-06-18 07:59:46 +01:00
CalDescent
5ac676d201 Added DataFileMessage and GetDataFileMessage, used for requesting and sending files between peers. 2021-06-17 19:05:00 +01:00
CalDescent
abfe0a925a More DataFile methods and improvements 2021-06-17 18:20:42 +01:00
CalDescent
fa11f4f45b Moved DataFileChunk(byte[] fileContent) constructor to the superclass so it can be used by regular data files too. 2021-06-17 18:20:11 +01:00
CalDescent
1e8dbfe4b7 Delete chunk if it fails the hash validation in the constructor. 2021-06-17 18:18:19 +01:00
CalDescent
f82f2bd287 Added DELETE /data/file API endpoint
This deletes a file referenced by a user supplied SHA256 digest string (which we will use as the file's "ID" in the Qortal data system). In the future this could be extended to delete all associated chunks, but first we need to build out the data chain so we have a way to look up chunks associated with a file hash.
2021-06-16 20:03:15 +01:00
CalDescent
76742c3869 Removed .dat extension, and use an extra level in the directory structure (data/aB/cD/aBcDeF... etc) 2021-06-16 19:49:12 +01:00
CalDescent
9407e7e418 Data storage location moved to settings ("dataPath") 2021-06-16 19:22:50 +01:00
CalDescent
120552b36e Added resource file missing from last commit. 2021-06-16 19:13:24 +01:00
CalDescent
9fb58c7ae3 Added the beginnings of an upload API, with some basic data file management. 2021-06-16 19:10:35 +01:00
CalDescent
b917da765c Removed block 212937 2021-06-15 09:29:07 +01:00
CalDescent
cc59510cd0 Removed all cross-chain code. It's not needed for data nodes. 2021-06-15 09:27:05 +01:00
CalDescent
86aab7023c Initial settings for data node development. Most to be decided later. 2021-06-14 19:40:37 +01:00
CalDescent
e3b0a41ba9 Merge branch 'sync-multiple-blocks' 2021-06-14 19:01:17 +01:00
CalDescent
802c55dfc8 Merge branch 'networking' 2021-06-14 19:00:17 +01:00
CalDescent
904be3005f Enable fast sync by default. 2021-06-12 13:07:25 +01:00
CalDescent
95eaf4c887 Merge branch 'master' into sync-multiple-blocks 2021-06-12 11:15:28 +01:00
CalDescent
bc6b3fb5f4 Include timestamps in block-timings.sh 2021-06-09 13:04:49 +01:00
CalDescent
df47f5d47b Merge branch 'master' into sync-multiple-blocks 2021-06-06 10:35:47 +01:00
CalDescent
c63a7884cb Limit to 10 untrimmed blocks per response, as they are larger than the trimmed ones. 2021-06-02 09:10:25 +01:00
CalDescent
cffbd41f26 Reduce memory allocations in onNetworkGetBlocksMessage 2021-06-02 09:09:06 +01:00
CalDescent
2ceba45782 Fast sync default blocks per request increased to 100. 2021-05-30 14:57:58 +01:00
CalDescent
ed423ed041 Increased MAX_DATA_SIZE and SYNC_BATCH_SIZE, to increase the effectiveness of the batch sync. 2021-05-30 14:54:13 +01:00
CalDescent
f58a52eaa4 Further work to increase the response timeout when requesting multiple blocks. 2021-05-30 13:10:38 +01:00
CalDescent
688404011b Relocate FETCH_BLOCKS_TIMEOUT to Peer.java and use a static import. 2021-05-30 10:08:50 +01:00
CalDescent
8881e0fb75 Merge branch 'master' into sync-multiple-blocks 2021-05-30 09:57:56 +01:00
CalDescent
61de7e144e Merge branch 'networking' into sync-multiple-blocks
# Conflicts:
#	src/main/java/org/qortal/network/Peer.java
2021-05-30 09:57:35 +01:00
CalDescent
7f5486dade Merge branch 'master' into sync-multiple-blocks 2021-05-25 07:37:01 +01:00
CalDescent
27aeb4f05f Merge branch 'master' into sync-multiple-blocks
# Conflicts:
#	src/main/java/org/qortal/controller/Synchronizer.java
2021-05-16 11:21:34 +01:00
CalDescent
255233fe38 Reduced log spam. 2021-05-10 09:10:51 +01:00
CalDescent
4ac3984b7c Merge branch 'master' into sync-multiple-blocks
# Conflicts:
#	src/main/java/org/qortal/settings/Settings.java
2021-05-10 09:09:41 +01:00
CalDescent
428af3c0e8 Only use fast sync on trimmed blocks, as others are too large.
This could probably be improved to make sure that all blocks in the next request are within the trimmed time range, but it's enough for now.
2021-05-09 13:57:47 +01:00
CalDescent
68544715bf Skip Block.logDebugInfo() altogether if the log level is more specific than DEBUG, to avoid wasting resources. 2021-05-09 09:05:51 +01:00
CalDescent
d2ea5633fb Fixed divide by zero exception.
Block.calcKeyDistance() cannot be called on some trimmed blocks, because the minter level is unable to be inferred in some cases. This generally hasn't been an issue, but the new Block.logDebugInfo() method is invoking it for all blocks. For now I am adding defensiveness to the debug method, but longer term we might want to add defensiveness to Block.calcKeyDistance() itself, if we ever encounter this issue again. I will leave it alone for now, to reduce risk.
2021-05-09 09:05:42 +01:00
CalDescent
3aa9b5f0b6 Increased timeout when syncing multiple blocks from 4s to 10s. 2021-05-08 22:36:41 +01:00
CalDescent
6c5dbf7bd0 Preliminary version for fast sync set to 1.6.0, because 1.5.x is already released. 2021-05-08 16:36:42 +01:00
CalDescent
3b3dc5032b Merge branch 'master' into sync-multiple-blocks
# Conflicts:
#	pom.xml
#	src/main/java/org/qortal/controller/Synchronizer.java

Removed all fast sync code from Controller.syncToPeerChain(), so it is now the same as `master`.
2021-05-08 16:35:09 +01:00
CalDescent
08f3d653cc Added new settings "maxBlocksPerRequest" and "maxBlocksPerResponse", to control the number of blocks requested and returned by nodes when using GetBlocksMessage and BlocksMessage. 2021-03-28 17:27:04 +01:00
CalDescent
f2bbafe6c2 Added missing break statement if a peer fails to respond with blocks when resolving a fork via fast sync. 2021-03-28 16:52:15 +01:00
CalDescent
cb80280eaf Bump Peer response timeout from 3s to 4s 2021-03-28 14:47:57 +01:00
CalDescent
f22f954ae3 Use MAXIMUM_BLOCKS_REQUEST_SIZE for GetBlocksMessage and BlockMessage, instead of MAXIMUM_REQUEST_SIZE.
Currently set to 1, as serialization of the BlocksMessage data on mainnet is too slow to use this for any significant number of blocks right now. Hopefully we can find a way to optimise this process, which will allow us to use this for multiple block syncing.

Until then, sticking with single blocks should still be enough to help solve the network congestion and re-orgs we are seeing, because it gives us the ability to request the next block based on the previous block's signature, which was unavailable using GET_BLOCK. This removes the requirement to fetch all block signatures upfront, and therefore it shouldn't matter if the peer does a partial re-org whilst a node is syncing to it.
2021-03-28 14:35:47 +01:00
CalDescent
2556855bd7 Added missing return statement if a peer fails to respond with blocks when fast syncing. 2021-03-28 14:19:53 +01:00
CalDescent
365662a2af MAXIMUM_RETRIES reduced from 3 to 1. It will now only retry once, which should save around 6 seconds of wasted synchronization time if a node is unable to respond with the requested block (due to a re-org, etc). 2021-03-27 19:25:27 +00:00
CalDescent
3e0ff7f43f Split Synchronizer.applyNewBlocks() into two methods.
If fast syncing is enabled in the settings (by default it's disabled) AND the peer is running at least v1.5.0, then it will route through to a new method which fetches multiple blocks at a time, in a very similar way to Synchronizer.syncToPeerChain().
If fast syncing is disabled in the settings, or we are communicating with a peer on an older version, it will continue to sync blocks individually.
2021-03-27 18:04:47 +00:00
CalDescent
8c3753326f Check "isFastSyncEnabledWhenResolvingFork" setting before requesting multiple blocks from a peer. This allows users to opt out of this functionality, by setting it to false in their settings. 2021-03-27 18:00:39 +00:00
CalDescent
dbcf6de2d5 Added new settings "fastSyncEnabled" (default: false) and "fastSyncEnabledWhenResolvingFork" (default: true). 2021-03-27 17:59:23 +00:00
CalDescent
a5308995b7 Bump version to 1.5.0, to allow nodes to start using the new syncing method. 2021-03-27 15:46:30 +00:00
CalDescent
270ac88b51 Added GetBlocksMessage and BlocksMessage, which allow multiple blocks to be transferred between peers in a single request.
When communicating with a peer that is running at least version 1.5.0, it will now sync multiple blocks at once in Synchronizer.syncToPeerChain(). This allows us to bypass the single block syncing (and retry mechanism), which has proven to be unviable when there are multiple active forks with several blocks in each chain.

For peers below v1.5.0, the logic should remain unaffected and it will continue to sync blocks individually.
2021-03-27 15:45:15 +00:00
CalDescent
e505067759 Added support for Native SegWit (Bech32) addresses, which have the prefixes "ltc1" and "bc1". Bitcoinj supports these automatically, as long as fromString() is used instead of fromBase58(), which is already the case. Tested on LTC mainnet, and BTC testnet. 2021-02-21 12:02:26 +00:00
866 changed files with 113309 additions and 14113 deletions

View File

@ -8,16 +8,16 @@ jobs:
mavenTesting:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Cache local Maven repository
uses: actions/cache@v2
uses: actions/cache@v3
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up the Java JDK
uses: actions/setup-java@v2
uses: actions/setup-java@v3
with:
java-version: '11'
distribution: 'adopt'

5
.gitignore vendored
View File

@ -14,7 +14,6 @@
/.mvn.classpath
/notes*
/settings.json
/testnet*
/settings*.json
/testchain*.json
/run-testnet*.sh
@ -28,3 +27,7 @@
/WindowsInstaller/Install Files/qortal.jar
/*.7z
/tmp
/wallets
/data*
/src/test/resources/arbitrary/*/.qortal/cache
apikey.txt

3
.vscode/settings.json vendored Normal file
View File

@ -0,0 +1,3 @@
{
"java.compile.nullAnalysis.mode": "automatic"
}

View File

@ -8,7 +8,7 @@
* Build auto-update download: `tools/build-auto-update.sh` - uploads auto-update file into new git branch
* Restart local node
* Publish auto-update transaction using *private key* for **non-admin** member of "dev" group:
`tools/publish-auto-update.sh non-admin-dev-member-private-key-in-base58`
`tools/publish-auto-update.pl non-admin-dev-member-private-key-in-base58`
* Wait for auto-update `ARBITRARY` transaction to be confirmed into a block
* Have "dev" group admins 'approve' auto-update using `tools/approve-auto-update.sh`
This tool will prompt for *private key* of **admin** of "dev" group

26
Dockerfile Normal file
View File

@ -0,0 +1,26 @@
FROM maven:3-openjdk-11 as builder
WORKDIR /work
COPY ./ /work/
RUN mvn clean package
###
FROM openjdk:11
RUN useradd -r -u 1000 -g users qortal && \
mkdir /usr/local/qortal /qortal && \
chown 1000:100 /qortal
COPY --from=builder /work/log4j2.properties /usr/local/qortal/
COPY --from=builder /work/target/qortal*.jar /usr/local/qortal/qortal.jar
USER 1000:100
EXPOSE 12391 12392
HEALTHCHECK --start-period=5m CMD curl -sf http://127.0.0.1:12391/admin/info || exit 1
WORKDIR /qortal
VOLUME /qortal
ENTRYPOINT ["java"]
CMD ["-Djava.net.preferIPv4Stack=false", "-jar", "/usr/local/qortal/qortal.jar"]

919
Q-Apps.md Normal file
View File

@ -0,0 +1,919 @@
# Qortal Project - Q-Apps Documentation
## Introduction
Q-Apps are static web apps written in javascript, HTML, CSS, and other static assets. The key difference between a Q-App and a fully static site is its ability to interact with both the logged-in user and on-chain data. This is achieved using the API described in this document.
# Section 0: Basic QDN concepts
## Introduction to QDN resources
Each published item on QDN (Qortal Data Network) is referred to as a "resource". A resource could contain anything from a few characters of text, to a multi-layered directory structure containing thousands of files.
Resources are stored on-chain, however the data payload is generally stored off-chain, and verified using an on-chain SHA-256 hash.
To publish a resource, a user must first register a name, and then include that name when publishing the data. Accounts without a registered name are unable to publish to QDN from a Q-App at this time.
Owning the name grants update privileges to the data. If that name is later sold or transferred, the permission to update that resource is moved to the new owner.
## Name, service & identifier
Each QDN resource has 3 important fields:
- `name` - the registered name of the account that is publishing the data (which will hold update/edit privileges going forwards).<br /><br />
- `service` - the type of content (e.g. IMAGE or JSON). Different services have different validation rules. See [list of available services](#services).<br /><br />
- `identifier` - an optional string to allow more than one resource to exist for a given name/service combination. For example, the name `QortalDemo` may wish to publish multiple images. This can be achieved by using a different identifier string for each. The identifier is only unique to the name in question, and so it doesn't matter if another name is using the same service and identifier string.
## Shared identifiers
Since an identifier can be used by multiple names, this can be used to the advantage of Q-App developers as it allows for data to be stored in a deterministic location.
An example of this is the user's avatar. This will always be published with service `THUMBNAIL` and identifier `qortal_avatar`, along with the user's name. So, an app can display the avatar of a user just by specifying their name when requesting the data. The same applies when publishing data.
## "Default" resources
A "default" resource refers to one without an identifier. For example, when a website is published via the UI, it will use the user's name and the service `WEBSITE`. These do not have an identifier, and are therefore the "default" website for this name. When requesting or publishing data without an identifier, apps can either omit the `identifier` key entirely, or include `"identifier": "default"` to indicate that the resource(s) being queried or published do not have an identifier.
<a name="services"></a>
## Available service types
Here is a list of currently available services that can be used in Q-Apps:
### Public services ###
The services below are intended to be used for publicly accessible data.
IMAGE,
THUMBNAIL,
VIDEO,
AUDIO,
PODCAST,
VOICE,
ARBITRARY_DATA,
JSON,
DOCUMENT,
LIST,
PLAYLIST,
METADATA,
BLOG,
BLOG_POST,
BLOG_COMMENT,
GIF_REPOSITORY,
ATTACHMENT,
FILE,
FILES,
CHAIN_DATA,
STORE,
PRODUCT,
OFFER,
COUPON,
CODE,
PLUGIN,
EXTENSION,
GAME,
ITEM,
NFT,
DATABASE,
SNAPSHOT,
COMMENT,
CHAIN_COMMENT,
WEBSITE,
APP,
QCHAT_ATTACHMENT,
QCHAT_IMAGE,
QCHAT_AUDIO,
QCHAT_VOICE
### Private services ###
For the services below, data is encrypted for a single recipient, and can only be decrypted using the private key of the recipient's wallet.
QCHAT_ATTACHMENT_PRIVATE,
ATTACHMENT_PRIVATE,
FILE_PRIVATE,
IMAGE_PRIVATE,
VIDEO_PRIVATE,
AUDIO_PRIVATE,
VOICE_PRIVATE,
DOCUMENT_PRIVATE,
MAIL_PRIVATE,
MESSAGE_PRIVATE
## Single vs multi-file resources
Some resources, such as those published with the `IMAGE` or `JSON` service, consist of a single file or piece of data (the image or the JSON string). This is the most common type of QDN resource, especially in the context of Q-Apps. These can be published by supplying a base64-encoded string containing the data.
Other resources, such as those published with the `WEBSITE`, `APP`, or `GIF_REPOSITORY` service, can contain multiple files and directories. Publishing these kinds of files is not yet available for Q-Apps, however it is possible to retrieve multi-file resources that are already published. When retrieving this data (via FETCH_QDN_RESOURCE), a `filepath` must be included to indicate the file that you would like to retrieve. There is no need to specify a filepath for single file resources, as these will automatically return the contents of the single file.
## App-specific data
Some apps may want to make all QDN data for a particular service available. However, others may prefer to only deal with data that has been published by their app (if a specific format/schema is being used for instance).
Identifiers can be used to allow app developers to locate data that has been published by their app. The recommended approach for this is to use the app name as a prefix on all identifiers when publishing data.
For instance, an app called `MyApp` could allow users to publish JSON data. The app could choose to prefix all identifiers with the string `myapp_`, and then use a random string for each published resource (resulting in identifiers such as `myapp_qR5ndZ8v`). Then, to locate data that has potentially been published by users of MyApp, it can later search the QDN database for items with `"service": "JSON"` and `"identifier": "myapp_"`. The SEARCH_QDN_RESOURCES action has a `prefix` option in order to match identifiers beginning with the supplied string.
Note that QDN is a permissionless system, and therefore it's not possible to verify that a resource was actually published by the app. It is recommended that apps validate the contents of the resource to ensure it is formatted correctly, instead of making assumptions.
## Updating a resource
To update a resource, it can be overwritten by publishing with the same `name`, `service`, and `identifier` combination. Note that the authenticated account must currently own the name in order to publish an update.
## Routing
If a non-existent `filepath` is accessed, the default behaviour of QDN is to return a `404: File not found` error. This includes anything published using the `WEBSITE` service.
However, routing is handled differently for anything published using the `APP` service.
For apps, QDN automatically sends all unhandled requests to the index file (generally index.html). This allows the app to use custom routing, as it is able to listen on any path. If a file exists at a path, the file itself will be served, and so the request won't be sent to the index file.
It's recommended that all apps return a 404 page if a request isn't able to be routed.
# Section 1: Simple links and image loading via HTML
## Section 1a: Linking to other QDN websites / resources
The `qortal://` protocol can be used to access QDN data from within Qortal websites and apps. The basic format is as follows:
```
<a href="qortal://{service}/{name}/{identifier}/{path}">link text</a>
```
However, the system will support the omission of the `identifier` and/or `path` components to allow for simpler URL formats.
A simple link to another website can be achieved with this HTML code:
```
<a href="qortal://WEBSITE/QortalDemo">link text</a>
```
To link to a specific page of another website:
```
<a href="qortal://WEBSITE/QortalDemo/minting-leveling/index.html">link text</a>
```
To link to a standalone resource, such as an avatar
```
<a href="qortal://THUMBNAIL/QortalDemo/qortal_avatar">avatar</a>
```
For cases where you would prefer to explicitly include an identifier (to remove ambiguity) you can use the keyword `default` to access a resource that doesn't have an identifier. For instance:
```
<a href="qortal://WEBSITE/QortalDemo/default">link to root of website</a>
<a href="qortal://WEBSITE/QortalDemo/default/minting-leveling/index.html">link to subpage of website</a>
```
## Section 1b: Linking to other QDN images
The same applies for images, such as displaying an avatar:
```
<img src="qortal://THUMBNAIL/QortalDemo/qortal_avatar" />
```
...or even an image from an entirely different website:
```
<img src="qortal://WEBSITE/AlphaX/assets/img/logo.png" />
```
# Section 2: Integrating a Javascript app
Javascript apps allow for much more complex integrations with Qortal's blockchain data.
## Section 2a: Direct API calls
The standard [Qortal Core API](http://localhost:12391/api-documentation) is available to websites and apps, and can be called directly using a standard AJAX request, such as:
```
async function getNameInfo(name) {
const response = await fetch("/names/" + name);
const nameData = await response.json();
console.log("nameData: " + JSON.stringify(nameData));
}
getNameInfo("QortalDemo");
```
However, this only works for read-only data, such as looking up transactions, names, balances, etc. Also, since the address of the logged in account can't be retrieved from the core, apps can't show personalized data with this approach.
## Section 2b: User interaction via qortalRequest()
To take things a step further, the qortalRequest() function can be used to interact with the user, in order to:
- Request address and public key of the logged in account
- Publish data to QDN
- Send chat messages
- Join groups
- Deploy ATs (smart contracts)
- Send QORT or any supported foreign coin
- Add/remove items from lists
In addition to the above, qortalRequest() also supports many read-only functions that are also available via direct core API calls. Using qortalRequest() helps with futureproofing, as the core APIs can be modified without breaking functionality of existing Q-Apps.
### Making a request
Qortal core will automatically inject the `qortalRequest()` javascript function to all websites/apps, which returns a Promise. This can be used to fetch data or publish data to the Qortal blockchain. This functionality supports async/await, as well as try/catch error handling.
```
async function myfunction() {
try {
let res = await qortalRequest({
action: "GET_ACCOUNT_DATA",
address: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2"
});
console.log(JSON.stringify(res)); // Log the response to the console
} catch(e) {
console.log("Error: " + JSON.stringify(e));
}
}
myfunction();
```
### Timeouts
Request timeouts are handled automatically when using qortalRequest(). The timeout value will differ based on the action being used - see `getDefaultTimeout()` in [q-apps.js](src/main/resources/q-apps/q-apps.js) for the current values.
If a request times out it will throw an error - `The request timed out` - which can be handled by the Q-App.
It is also possible to specify a custom timeout using `qortalRequestWithTimeout(request, timeout)`, however this is discouraged. It's more reliable and futureproof to let the core handle the timeout values.
# Section 3: qortalRequest Documentation
## Supported actions
Here is a list of currently supported actions:
- GET_USER_ACCOUNT
- GET_ACCOUNT_DATA
- GET_ACCOUNT_NAMES
- SEARCH_NAMES
- GET_NAME_DATA
- LIST_QDN_RESOURCES
- SEARCH_QDN_RESOURCES
- GET_QDN_RESOURCE_STATUS
- GET_QDN_RESOURCE_PROPERTIES
- GET_QDN_RESOURCE_METADATA
- GET_QDN_RESOURCE_URL
- LINK_TO_QDN_RESOURCE
- FETCH_QDN_RESOURCE
- PUBLISH_QDN_RESOURCE
- PUBLISH_MULTIPLE_QDN_RESOURCES
- DECRYPT_DATA
- SAVE_FILE
- GET_WALLET_BALANCE
- GET_BALANCE
- SEND_COIN
- SEARCH_CHAT_MESSAGES
- SEND_CHAT_MESSAGE
- LIST_GROUPS
- JOIN_GROUP
- DEPLOY_AT
- GET_AT
- GET_AT_DATA
- LIST_ATS
- FETCH_BLOCK
- FETCH_BLOCK_RANGE
- SEARCH_TRANSACTIONS
- GET_PRICE
- GET_LIST_ITEMS
- ADD_LIST_ITEMS
- DELETE_LIST_ITEM
More functionality will be added in the future.
## Example Requests
Here are some example requests for each of the above:
### Get address of logged in account
_Will likely require user approval_
```
let account = await qortalRequest({
action: "GET_USER_ACCOUNT"
});
let address = account.address;
```
### Get public key of logged in account
_Will likely require user approval_
```
let pubkey = await qortalRequest({
action: "GET_USER_ACCOUNT"
});
let publicKey = account.publicKey;
```
### Get account data
```
let res = await qortalRequest({
action: "GET_ACCOUNT_DATA",
address: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2"
});
```
### Get names owned by account
```
let res = await qortalRequest({
action: "GET_ACCOUNT_NAMES",
address: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2"
});
```
### Search names
```
let res = await qortalRequest({
action: "SEARCH_NAMES",
query: "search query goes here",
prefix: false, // Optional - if true, only the beginning of the name is matched
limit: 100,
offset: 0,
reverse: false
});
```
### Get name data
```
let res = await qortalRequest({
action: "GET_NAME_DATA",
name: "QortalDemo"
});
```
### List QDN resources
```
let res = await qortalRequest({
action: "LIST_QDN_RESOURCES",
service: "THUMBNAIL",
name: "QortalDemo", // Optional (exact match)
identifier: "qortal_avatar", // Optional (exact match)
default: true, // Optional
includeStatus: false, // Optional - will take time to respond, so only request if necessary
includeMetadata: false, // Optional - will take time to respond, so only request if necessary
followedOnly: false, // Optional - include followed names only
excludeBlocked: false, // Optional - exclude blocked content
limit: 100,
offset: 0,
reverse: true
});
```
### Search QDN resources
```
let res = await qortalRequest({
action: "SEARCH_QDN_RESOURCES",
service: "THUMBNAIL",
query: "search query goes here", // Optional - searches both "identifier" and "name" fields
identifier: "search query goes here", // Optional - searches only the "identifier" field
name: "search query goes here", // Optional - searches only the "name" field
prefix: false, // Optional - if true, only the beginning of fields are matched in all of the above filters
exactMatchNames: true, // Optional - if true, partial name matches are excluded
default: false, // Optional - if true, only resources without identifiers are returned
mode: "LATEST", // Optional - whether to return all resources or just the latest for a name/service combination. Possible values: ALL,LATEST. Default: LATEST
minLevel: 1, // Optional - whether to filter results by minimum account level
includeStatus: false, // Optional - will take time to respond, so only request if necessary
includeMetadata: false, // Optional - will take time to respond, so only request if necessary
nameListFilter: "QApp1234Subscriptions", // Optional - will only return results if they are from a name included in supplied list
followedOnly: false, // Optional - include followed names only
excludeBlocked: false, // Optional - exclude blocked content
// before: 1683546000000, // Optional - limit to resources created before timestamp
// after: 1683546000000, // Optional - limit to resources created after timestamp
limit: 100,
offset: 0,
reverse: true
});
```
### Search QDN resources (multiple names)
```
let res = await qortalRequest({
action: "SEARCH_QDN_RESOURCES",
service: "THUMBNAIL",
query: "search query goes here", // Optional - searches both "identifier" and "name" fields
identifier: "search query goes here", // Optional - searches only the "identifier" field
names: ["QortalDemo", "crowetic", "AlphaX"], // Optional - searches only the "name" field for any of the supplied names
prefix: false, // Optional - if true, only the beginning of fields are matched in all of the above filters
exactMatchNames: true, // Optional - if true, partial name matches are excluded
default: false, // Optional - if true, only resources without identifiers are returned
mode: "LATEST", // Optional - whether to return all resources or just the latest for a name/service combination. Possible values: ALL,LATEST. Default: LATEST
includeStatus: false, // Optional - will take time to respond, so only request if necessary
includeMetadata: false, // Optional - will take time to respond, so only request if necessary
nameListFilter: "QApp1234Subscriptions", // Optional - will only return results if they are from a name included in supplied list
followedOnly: false, // Optional - include followed names only
excludeBlocked: false, // Optional - exclude blocked content
// before: 1683546000000, // Optional - limit to resources created before timestamp
// after: 1683546000000, // Optional - limit to resources created after timestamp
limit: 100,
offset: 0,
reverse: true
});
```
### Fetch QDN single file resource
```
let res = await qortalRequest({
action: "FETCH_QDN_RESOURCE",
name: "QortalDemo",
service: "THUMBNAIL",
identifier: "qortal_avatar", // Optional. If omitted, the default resource is returned, or you can alternatively use the keyword "default"
encoding: "base64", // Optional. If omitted, data is returned in raw form
rebuild: false
});
```
### Fetch file from multi file QDN resource
Data is returned in the base64 format
```
let res = await qortalRequest({
action: "FETCH_QDN_RESOURCE",
name: "QortalDemo",
service: "WEBSITE",
identifier: "default", // Optional. If omitted, the default resource is returned, or you can alternatively request that using the keyword "default", as shown here
filepath: "index.html", // Required only for resources containing more than one file
rebuild: false
});
```
### Get QDN resource status
```
let res = await qortalRequest({
action: "GET_QDN_RESOURCE_STATUS",
name: "QortalDemo",
service: "THUMBNAIL",
identifier: "qortal_avatar", // Optional
build: true // Optional - request that the resource is fetched & built in the background
});
```
### Get QDN resource properties
```
let res = await qortalRequest({
action: "GET_QDN_RESOURCE_PROPERTIES",
name: "QortalDemo",
service: "THUMBNAIL",
identifier: "qortal_avatar" // Optional
});
// Returns: filename, size, mimeType (where available)
```
### Get QDN resource metadata
```
let res = await qortalRequest({
action: "GET_QDN_RESOURCE_METADATA",
name: "QortalDemo",
service: "THUMBNAIL",
identifier: "qortal_avatar" // Optional
});
```
### Publish a single file to QDN
_Requires user approval_.<br />
Note: this publishes a single, base64-encoded file. Multi-file resource publishing (such as a WEBSITE or GIF_REPOSITORY) is not yet supported via a Q-App. It will be added in a future update.
```
let res = await qortalRequest({
action: "PUBLISH_QDN_RESOURCE",
name: "Demo", // Publisher must own the registered name - use GET_ACCOUNT_NAMES for a list
service: "IMAGE",
identifier: "myapp-image1234" // Optional
data64: "base64_encoded_data",
// filename: "image.jpg", // Optional - to help apps determine the file's type
// title: "Title", // Optional
// description: "Description", // Optional
// category: "TECHNOLOGY", // Optional
// tag1: "any", // Optional
// tag2: "strings", // Optional
// tag3: "can", // Optional
// tag4: "go", // Optional
// tag5: "here", // Optional
// encrypt: true, // Optional - to be used with a private service
// recipientPublicKey: "publickeygoeshere" // Only required if `encrypt` is set to true
});
```
### Publish multiple resources at once to QDN
_Requires user approval_.<br />
Note: each resource being published consists of a single, base64-encoded file, each in its own transaction. Useful for publishing two or more related things, such as a video and a video thumbnail.
```
let res = await qortalRequest({
action: "PUBLISH_MULTIPLE_QDN_RESOURCES",
resources: [
name: "Demo", // Publisher must own the registered name - use GET_ACCOUNT_NAMES for a list
service: "IMAGE",
identifier: "myapp-image1234" // Optional
data64: "base64_encoded_data",
// filename: "image.jpg", // Optional - to help apps determine the file's type
// title: "Title", // Optional
// description: "Description", // Optional
// category: "TECHNOLOGY", // Optional
// tag1: "any", // Optional
// tag2: "strings", // Optional
// tag3: "can", // Optional
// tag4: "go", // Optional
// tag5: "here", // Optional
// encrypt: true, // Optional - to be used with a private service
// recipientPublicKey: "publickeygoeshere" // Only required if `encrypt` is set to true
],
[
... more resources here if needed ...
]
});
```
### Decrypt encrypted/private data
```
let res = await qortalRequest({
action: "DECRYPT_DATA",
encryptedData: 'qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1r',
publicKey: 'publickeygoeshere'
});
// Returns base64 encoded string of plaintext data
```
### Prompt user to save a file to disk
Note: mimeType not required but recommended. If not specified, saving will fail if the mimeType is unable to be derived from the Blob.
```
let res = await qortalRequest({
action: "SAVE_FILE",
blob: dataBlob,
filename: "myfile.pdf",
mimeType: "application/pdf" // Optional but recommended
});
```
### Get wallet balance (QORT)
_Requires user approval_
```
let res = await qortalRequest({
action: "GET_WALLET_BALANCE",
coin: "QORT"
});
```
### Get address or asset balance
```
let res = await qortalRequest({
action: "GET_BALANCE",
address: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2"
});
```
```
let res = await qortalRequest({
action: "GET_BALANCE",
assetId: 1,
address: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2"
});
```
### Send QORT to address
_Requires user approval_
```
let res = await qortalRequest({
action: "SEND_COIN",
coin: "QORT",
destinationAddress: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2",
amount: 1.00000000 // 1 QORT
});
```
### Send foreign coin to address
_Requires user approval_<br />
Note: default fees can be found [here](https://github.com/Qortal/qortal-ui/blob/master/plugins/plugins/core/qdn/browser/browser.src.js#L205-L209).
```
let res = await qortalRequest({
action: "SEND_COIN",
coin: "LTC",
destinationAddress: "LSdTvMHRm8sScqwCi6x9wzYQae8JeZhx6y",
amount: 1.00000000, // 1 LTC
fee: 0.00000020 // Optional fee per byte (default fee used if omitted, recommended) - not used for QORT or ARRR
});
```
### Search or list chat messages
```
let res = await qortalRequest({
action: "SEARCH_CHAT_MESSAGES",
before: 999999999999999,
after: 0,
txGroupId: 0, // Optional (must specify either txGroupId or two involving addresses)
// involving: ["QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2", "QSefrppsDCsZebcwrqiM1gNbWq7YMDXtG2"], // Optional (must specify either txGroupId or two involving addresses)
// reference: "reference", // Optional
// chatReference: "chatreference", // Optional
// hasChatReference: true, // Optional
encoding: "BASE64", // Optional (defaults to BASE58 if omitted)
limit: 100,
offset: 0,
reverse: true
});
```
### Send a group chat message
_Requires user approval_
```
let res = await qortalRequest({
action: "SEND_CHAT_MESSAGE",
groupId: 0,
message: "Test"
});
```
### Send a private chat message
_Requires user approval_
```
let res = await qortalRequest({
action: "SEND_CHAT_MESSAGE",
destinationAddress: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2",
message: "Test"
});
```
### List groups
```
let res = await qortalRequest({
action: "LIST_GROUPS",
limit: 100,
offset: 0,
reverse: true
});
```
### Join a group
_Requires user approval_
```
let res = await qortalRequest({
action: "JOIN_GROUP",
groupId: 100
});
```
### Deploy an AT
_Requires user approval_
```
let res = await qortalRequest({
action: "DEPLOY_AT",
creationBytes: "12345", // Must be Base58 encoded
name: "test name",
description: "test description",
type: "test type",
tags: "test tags",
amount: 1.00000000, // 1 QORT
assetId: 0,
// fee: 0.002 // optional - will use default fee if excluded
});
```
### Get AT info
```
let res = await qortalRequest({
action: "GET_AT",
atAddress: "ASRUsCjk6fa5bujv3oWYmWaVqNtvxydpPH"
});
```
### Get AT data bytes (base58 encoded)
```
let res = await qortalRequest({
action: "GET_AT_DATA",
atAddress: "ASRUsCjk6fa5bujv3oWYmWaVqNtvxydpPH"
});
```
### List ATs by functionality
```
let res = await qortalRequest({
action: "LIST_ATS",
codeHash58: "4KdJETRAdymE7dodDmJbf5d9L1bp4g5Nxky8m47TBkvA",
isExecutable: true,
limit: 100,
offset: 0,
reverse: true
});
```
### Fetch block by signature
```
let res = await qortalRequest({
action: "FETCH_BLOCK",
signature: "875yGFUy1zHV2hmxNWzrhtn9S1zkeD7SQppwdXFysvTXrankCHCz4iyAUgCBM3GjvibbnyRQpriuy1cyu953U1u5uQdzuH3QjQivi9UVwz86z1Akn17MGd5Z5STjpDT7248K6vzMamuqDei57Znonr8GGgn8yyyABn35CbZUCeAuXju"
});
```
### Fetch block by height
```
let res = await qortalRequest({
action: "FETCH_BLOCK",
height: "1139850"
});
```
### Fetch a range of blocks
```
let res = await qortalRequest({
action: "FETCH_BLOCK_RANGE",
height: "1139800",
count: 20,
reverse: false
});
```
### Search transactions
```
let res = await qortalRequest({
action: "SEARCH_TRANSACTIONS",
// startBlock: 1139000,
// blockLimit: 1000,
txGroupId: 0,
txType: [
"PAYMENT",
"REWARD_SHARE"
],
confirmationStatus: "CONFIRMED",
limit: 10,
offset: 0,
reverse: false
});
```
### Get an estimate of the QORT price
```
let res = await qortalRequest({
action: "GET_PRICE",
blockchain: "LITECOIN",
// maxtrades: 10,
inverse: true
});
```
### Get URL to load a QDN resource
Note: this returns a "Resource does not exist" error if a non-existent resource is requested.
```
let url = await qortalRequest({
action: "GET_QDN_RESOURCE_URL",
service: "THUMBNAIL",
name: "QortalDemo",
identifier: "qortal_avatar"
// path: "filename.jpg" // optional - not needed if resource contains only one file
});
```
### Get URL to load a QDN website
Note: this returns a "Resource does not exist" error if a non-existent resource is requested.
```
let url = await qortalRequest({
action: "GET_QDN_RESOURCE_URL",
service: "WEBSITE",
name: "QortalDemo",
});
```
### Get URL to load a specific file from a QDN website
Note: this returns a "Resource does not exist" error if a non-existent resource is requested.
```
let url = await qortalRequest({
action: "GET_QDN_RESOURCE_URL",
service: "WEBSITE",
name: "AlphaX",
path: "/assets/img/logo.png"
});
```
### Link/redirect to another QDN website
Note: an alternate method is to include `<a href="qortal://WEBSITE/QortalDemo">link text</a>` within your HTML code.
```
let res = await qortalRequest({
action: "LINK_TO_QDN_RESOURCE",
service: "WEBSITE",
name: "QortalDemo",
});
```
### Link/redirect to a specific path of another QDN website
Note: an alternate method is to include `<a href="qortal://WEBSITE/QortalDemo/minting-leveling/index.html">link text</a>` within your HTML code.
```
let res = await qortalRequest({
action: "LINK_TO_QDN_RESOURCE",
service: "WEBSITE",
name: "QortalDemo",
path: "/minting-leveling/index.html"
});
```
### Get the contents of a list
_Requires user approval_
```
let res = await qortalRequest({
action: "GET_LIST_ITEMS",
list_name: "followedNames"
});
```
### Add one or more items to a list
_Requires user approval_
```
let res = await qortalRequest({
action: "ADD_LIST_ITEMS",
list_name: "blockedNames",
items: ["QortalDemo"]
});
```
### Delete a single item from a list
_Requires user approval_.
Items must be deleted one at a time.
```
let res = await qortalRequest({
action: "DELETE_LIST_ITEM",
list_name: "blockedNames",
item: "QortalDemo"
});
```
# Section 4: Examples
Some example projects can be found [here](https://github.com/Qortal/Q-Apps). These can be cloned and modified, or used as a reference when creating a new app.
## Sample App
Here is a sample application to display the logged-in user's avatar:
```
<html>
<head>
<script>
async function showAvatar() {
try {
// Get QORT address of logged in account
let account = await qortalRequest({
action: "GET_USER_ACCOUNT"
});
let address = account.address;
console.log("address: " + address);
// Get names owned by this account
let names = await qortalRequest({
action: "GET_ACCOUNT_NAMES",
address: address
});
console.log("names: " + JSON.stringify(names));
if (names.length == 0) {
console.log("User has no registered names");
return;
}
// Download base64-encoded avatar of the first registered name
let avatar = await qortalRequest({
action: "FETCH_QDN_RESOURCE",
name: names[0].name,
service: "THUMBNAIL",
identifier: "qortal_avatar",
encoding: "base64"
});
console.log("Avatar size: " + avatar.length + " bytes");
// Display the avatar image on the screen
document.getElementById("avatar").src = "data:image/png;base64," + avatar;
} catch(e) {
console.log("Error: " + JSON.stringify(e));
}
}
</script>
</head>
<body onload="showAvatar()">
<img width="500" id="avatar" />
</body>
</html>
```
# Section 5: Testing and Development
Publishing an in-development app to mainnet isn't recommended. There are several options for developing and testing a Q-app before publishing to mainnet:
### Preview mode
Select "Preview" in the UI after choosing the zip. This allows for full Q-App testing without the need to publish any data.
### Testnets
For an end-to-end test of Q-App publishing, you can use the official testnet, or set up a single node testnet of your own (often referred to as devnet) on your local machine. See [Single Node Testnet Quick Start Guide](testnet/README.md#quick-start).
### Debugging
It is recommended that you develop and test in a web browser, to allow access to the javascript console. To do this:
1. Open the UI app, then minimise it.
2. In a Chromium-based web browser, visit: http://localhost:12388/
3. Log in to your account and then preview your app/website.
4. Go to `View > Developer > JavaScript Console`. Here you can monitor console logs, errors, and network requests from your app, in the same way as any other web-app.

View File

@ -1,69 +0,0 @@
# How to build a testnet
## Create testnet blockchain config
- You can begin by copying the mainnet blockchain config `src/main/resources/blockchain.json`
- Insert `"isTestChain": true,` after the opening `{`
- Modify testnet genesis block
### Testnet genesis block
- Set `timestamp` to a nearby future value, e.g. 15 mins from 'now'
This is to give yourself enough time to set up other testnet nodes
- Retain the initial `ISSUE_ASSET` transactions!
- Add `ACCOUNT_FLAGS` transactions with `"andMask": -1, "orMask": 1, "xorMask": 0` to create founders
- Add at least one `REWARD_SHARE` transaction otherwise no-one can mint initial blocks!
You will need to calculate `rewardSharePublicKey` (and private key),
or make a new account on mainnet and use self-share key values
- Add `ACCOUNT_LEVEL` transactions to set initial level of accounts as needed
- Add `GENESIS` transactions to add QORT/LEGACY_QORA funds to accounts as needed
## Testnet `settings.json`
- Create a new `settings-test.json`
- Make sure to add `"isTestNet": true,`
- Make sure to reference testnet blockchain config file: `"blockchainConfig": "testchain.json",`
- It is a good idea to use a separate database: `"repositoryPath": "db-testnet",`
- You might also need to add `"bitcoinNet": "TEST3",` and `"litecoinNet": "TEST3",`
## Other nodes
- Copy `testchain.json` and `settings-test.json` to other nodes
- Alternatively, you can run multiple nodes on the same machine by:
* Copying `settings-test.json` to `settings-test-1.json`
* Configure different `repositoryPath`
* Configure use of different ports:
+ `"apiPort": 22391,`
+ `"listenPort": 22392,`
## Starting-up
- Start up at least as many nodes as `minBlockchainPeers` (or adjust this value instead)
- Probably best to perform API call `DELETE /peers/known`
- Add other nodes via API call `POST /peers <peer-hostname-or-IP>`
- Add minting private key to node(s) via API call `POST /admin/mintingaccounts <minting-private-key>`
This key must have corresponding `REWARD_SHARE` transaction in testnet genesis block
- Wait for genesis block timestamp to pass
- A node should mint block 2 approximately 60 seconds after genesis block timestamp
- Other testnet nodes will sync *as long as there is at least `minBlockchainPeers` peers with an "up-to-date" chain`
- You can also use API call `POST /admin/forcesync <connected-peer-IP-and-port>` on stuck nodes
## Dealing with stuck chain
Maybe your nodes have been offline and no-one has minted a recent testnet block.
Your options are:
- Start a new testnet from scratch
- Fire up your testnet node(s)
- Force one of your nodes to mint by:
+ Set a debugger breakpoint on Settings.getMinBlockchainPeers()
+ When breakpoint is hit, change `this.minBlockchainPeers` to zero, then continue
- Once one of your nodes has minted blocks up to 'now', you can use "forcesync" on the other nodes
## Tools
- `qort` tool, but use `-t` option for default testnet API port (62391)
- `qort` tool, but first set shell variable: `export BASE_URL=some-node-hostname-or-ip:port`
- `qort` tool, but prepend with one-time shell variable: `BASE_URL=some-node-hostname-or-ip:port qort ......`
- `peer-heights`, but use `-t` option, or `BASE_URL` shell variable as above

View File

@ -1,7 +1,7 @@
rootLogger.level = info
# On Windows, uncomment next line to set dirname:
# property.dirname = ${sys:user.home}\\AppData\\Local\\qortal\\
property.filename = ${sys:log4j2.filenameTemplate:-log.txt}
# property.filename = ${sys:log4j2.filenameTemplate:-log.txt}
rootLogger.appenderRef.console.ref = stdout
rootLogger.appenderRef.rolling.ref = FILE
@ -59,11 +59,14 @@ appender.console.filter.threshold.level = error
appender.rolling.type = RollingFile
appender.rolling.name = FILE
appender.rolling.fileName = qortal.log
appender.rolling.filePattern = qortal.%d{dd-MMM}.log.gz
appender.rolling.layout.type = PatternLayout
appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
appender.rolling.filePattern = ${dirname:-}${filename}.%i
appender.rolling.policy.type = SizeBasedTriggeringPolicy
appender.rolling.policy.size = 4MB
appender.rolling.policy.size = 10MB
appender.rolling.strategy.type = DefaultRolloverStrategy
appender.rolling.strategy.max = 7
# Set the immediate flush to true (default)
# appender.rolling.immediateFlush = true
# Set the append to true (default), should not overwrite

File diff suppressed because it is too large Load Diff

View File

@ -2,8 +2,8 @@
## Prerequisites
* AdvancedInstaller v16 or better, and enterprise licence if translations are required
* Installed AdoptOpenJDK v11 64bit, full JDK *not* JRE
* AdvancedInstaller v19.4 or better, and enterprise licence if translations are required
* Installed AdoptOpenJDK v17 64bit, full JDK *not* JRE
## General build instructions
@ -15,10 +15,8 @@ Typical build procedure:
* Place the `qortal.jar` file in `Install-Files\`
* Open AdvancedInstaller with qortal.aip file
* If releasing a new version, change version number in:
+ "Product Information" side menu
+ "Product Details" side menu entry
+ "Product Details" tab in "Product Details" pane
+ "Product Version" entry box
* Click away to a different side menu entry, e.g. "Resources" -> "Files and Folders"
* You should be prompted whether to generate a new product key, click "Generate New"
* Click "Build" button

Binary file not shown.

View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<groupId>com.dosse</groupId>
<artifactId>WaifUPnP</artifactId>
<version>1.1</version>
<description>POM was created from install:install-file</description>
</project>

Binary file not shown.

View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<groupId>com.dosse</groupId>
<artifactId>WaifUPnP</artifactId>
<version>1.2</version>
<description>POM was created from install:install-file</description>
</project>

View File

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<metadata>
<groupId>com.dosse</groupId>
<artifactId>WaifUPnP</artifactId>
<versioning>
<release>1.2</release>
<versions>
<version>1.1</version>
<version>1.2</version>
</versions>
<lastUpdated>20231026200127</lastUpdated>
</versioning>
</metadata>

Binary file not shown.

View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<groupId>org.ciyam</groupId>
<artifactId>AT</artifactId>
<version>1.4.0</version>
<description>POM was created from install:install-file</description>
</project>

Binary file not shown.

View File

@ -0,0 +1,123 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.ciyam</groupId>
<artifactId>AT</artifactId>
<version>1.4.1</version>
<packaging>jar</packaging>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<skipTests>false</skipTests>
<bouncycastle.version>1.69</bouncycastle.version>
<junit.version>4.13.2</junit.version>
<maven-compiler-plugin.version>3.11.0</maven-compiler-plugin.version>
<maven-jar-plugin.version>3.3.0</maven-jar-plugin.version>
<maven-javadoc-plugin.version>3.6.3</maven-javadoc-plugin.version>
<maven-source-plugin.version>3.3.0</maven-source-plugin.version>
<maven-surefire-plugin.version>3.2.2</maven-surefire-plugin.version>
</properties>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
<testSourceDirectory>src/test/java</testSourceDirectory>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin.version}</version>
<configuration>
<source>11</source>
<target>11</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven-surefire-plugin.version}</version>
<configuration>
<skipTests>${skipTests}</skipTests>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>${maven-source-plugin.version}</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>${maven-javadoc-plugin.version}</version>
<executions>
<execution>
<id>attach-javadoc</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin.version}</version>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven-surefire-plugin.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>${maven-source-plugin.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>${maven-javadoc-plugin.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin.version}</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
<version>${bouncycastle.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

Binary file not shown.

View File

@ -0,0 +1,123 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.ciyam</groupId>
<artifactId>AT</artifactId>
<version>1.4.2</version>
<packaging>jar</packaging>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<skipTests>false</skipTests>
<bouncycastle.version>1.70</bouncycastle.version>
<junit.version>4.13.2</junit.version>
<maven-compiler-plugin.version>3.13.0</maven-compiler-plugin.version>
<maven-source-plugin.version>3.3.0</maven-source-plugin.version>
<maven-javadoc-plugin.version>3.6.3</maven-javadoc-plugin.version>
<maven-surefire-plugin.version>3.2.5</maven-surefire-plugin.version>
<maven-jar-plugin.version>3.4.1</maven-jar-plugin.version>
</properties>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
<testSourceDirectory>src/test/java</testSourceDirectory>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin.version}</version>
<configuration>
<source>11</source>
<target>11</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven-surefire-plugin.version}</version>
<configuration>
<skipTests>${skipTests}</skipTests>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>${maven-source-plugin.version}</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>${maven-javadoc-plugin.version}</version>
<executions>
<execution>
<id>attach-javadoc</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin.version}</version>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven-surefire-plugin.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>${maven-source-plugin.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>${maven-javadoc-plugin.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin.version}</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
<version>${bouncycastle.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@ -3,14 +3,14 @@
<groupId>org.ciyam</groupId>
<artifactId>AT</artifactId>
<versioning>
<release>1.3.8</release>
<release>1.4.2</release>
<versions>
<version>1.3.4</version>
<version>1.3.5</version>
<version>1.3.6</version>
<version>1.3.7</version>
<version>1.3.8</version>
<version>1.4.0</version>
<version>1.4.1</version>
<version>1.4.2</version>
</versions>
<lastUpdated>20200925114415</lastUpdated>
<lastUpdated>20240426084210</lastUpdated>
</versioning>
</metadata>

View File

@ -1,7 +1,7 @@
rootLogger.level = info
# On Windows, uncomment next line to set dirname:
# property.dirname = ${sys:user.home}\\AppData\\Local\\qortal\\
property.filename = ${sys:log4j2.filenameTemplate:-log.txt}
# property.filename = ${sys:log4j2.filenameTemplate:-log.txt}
rootLogger.appenderRef.console.ref = stdout
rootLogger.appenderRef.rolling.ref = FILE
@ -59,11 +59,14 @@ appender.console.filter.threshold.level = error
appender.rolling.type = RollingFile
appender.rolling.name = FILE
appender.rolling.fileName = qortal.log
appender.rolling.filePattern = qortal.%d{dd-MMM}.log.gz
appender.rolling.layout.type = PatternLayout
appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
appender.rolling.filePattern = ${dirname:-}${filename}.%i
appender.rolling.policy.type = SizeBasedTriggeringPolicy
appender.rolling.policy.size = 4MB
appender.rolling.policy.size = 10MB
appender.rolling.strategy.type = DefaultRolloverStrategy
appender.rolling.strategy.max = 7
# Set the immediate flush to true (default)
# appender.rolling.immediateFlush = true
# Set the append to true (default), should not overwrite

261
pom.xml
View File

@ -3,31 +3,61 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.qortal</groupId>
<artifactId>qortal</artifactId>
<version>2.1.3</version>
<version>4.5.2</version>
<packaging>jar</packaging>
<properties>
<skipTests>true</skipTests>
<altcoinj.version>bf9fb80</altcoinj.version>
<bitcoinj.version>0.15.10</bitcoinj.version>
<bouncycastle.version>1.64</bouncycastle.version>
<build.timestamp>${maven.build.timestamp}</build.timestamp>
<ciyam-at.version>1.3.8</ciyam-at.version>
<commons-net.version>3.6</commons-net.version>
<commons-text.version>1.8</commons-text.version>
<commons-io.version>2.6</commons-io.version>
<commons-compress.version>1.21</commons-compress.version>
<xz.version>1.9</xz.version>
<dagger.version>1.2.2</dagger.version>
<guava.version>28.1-jre</guava.version>
<hsqldb.version>2.5.1</hsqldb.version>
<jersey.version>2.29.1</jersey.version>
<jetty.version>9.4.29.v20200521</jetty.version>
<log4j.version>2.12.1</log4j.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<slf4j.version>1.7.12</slf4j.version>
<swagger-api.version>2.0.9</swagger-api.version>
<swagger-ui.version>3.23.8</swagger-ui.version>
<package-info-maven-plugin.version>1.1.0</package-info-maven-plugin.version>
<skipTests>true</skipTests>
<altcoinj.version>7dc8c6f</altcoinj.version>
<bitcoinj.version>0.15.10</bitcoinj.version>
<bouncycastle.version>1.70</bouncycastle.version>
<build.timestamp>${maven.build.timestamp}</build.timestamp>
<ciyam-at.version>1.4.2</ciyam-at.version>
<commons-net.version>3.8.0</commons-net.version>
<commons-text.version>1.12.0</commons-text.version>
<commons-io.version>2.16.1</commons-io.version>
<commons-compress.version>1.26.2</commons-compress.version>
<commons-lang3.version>3.14.0</commons-lang3.version>
<dagger.version>1.2.2</dagger.version>
<extendedset.version>0.12.3</extendedset.version>
<git-commit-id-plugin.version>4.9.10</git-commit-id-plugin.version>
<grpc.version>1.65.0</grpc.version>
<guava.version>33.2.1-jre</guava.version>
<hamcrest-library.version>2.2</hamcrest-library.version>
<homoglyph.version>1.2.1</homoglyph.version>
<hsqldb.version>2.5.1</hsqldb.version>
<icu4j.version>75.1</icu4j.version>
<java-diff-utils.version>4.12</java-diff-utils.version>
<javax.servlet-api.version>4.0.1</javax.servlet-api.version>
<jaxb-runtime.version>2.3.9</jaxb-runtime.version>
<jersey.version>2.42</jersey.version>
<jetty.version>9.4.54.v20240208</jetty.version>
<json-simple.version>1.1.1</json-simple.version>
<json.version>20240303</json.version>
<jsoup.version>1.17.2</jsoup.version>
<junit-jupiter-engine.version>5.11.0-M2</junit-jupiter-engine.version>
<lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
<log4j.version>2.23.1</log4j.version>
<mail.version>1.5.0-b01</mail.version>
<maven-build-helper-plugin.version>3.6.0</maven-build-helper-plugin.version>
<maven-compiler-plugin.version>3.13.0</maven-compiler-plugin.version>
<maven-dependency-plugin.version>3.6.1</maven-dependency-plugin.version>
<maven-jar-plugin.version>3.4.2</maven-jar-plugin.version>
<maven-package-info-plugin.version>1.1.0</maven-package-info-plugin.version>
<maven-plugin.version>2.16.2</maven-plugin.version>
<maven-reproducible-build-plugin.version>0.16</maven-reproducible-build-plugin.version>
<maven-resources-plugin.version>3.3.1</maven-resources-plugin.version>
<maven-shade-plugin.version>3.6.0</maven-shade-plugin.version>
<maven-surefire-plugin.version>3.3.0</maven-surefire-plugin.version>
<protobuf.version>3.25.3</protobuf.version>
<replacer.version>1.5.3</replacer.version>
<simplemagic.version>1.17</simplemagic.version>
<slf4j.version>1.7.36</slf4j.version>
<swagger-api.version>2.0.10</swagger-api.version>
<swagger-ui.version>5.17.14</swagger-ui.version>
<upnp.version>1.2</upnp.version>
<xz.version>1.9</xz.version>
</properties>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
@ -42,14 +72,14 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
<version>2.5</version>
<version>${maven-plugin.version}</version>
<configuration>
<generateBackupPoms>false</generateBackupPoms>
</configuration>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<version>${maven-compiler-plugin.version}</version>
<configuration>
<release>11</release>
</configuration>
@ -80,7 +110,7 @@
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>4.0.0</version>
<version>${git-commit-id-plugin.version}</version>
<executions>
<execution>
<id>get-the-git-infos</id>
@ -112,7 +142,7 @@
<plugin>
<groupId>com.google.code.maven-replacer-plugin</groupId>
<artifactId>replacer</artifactId>
<version>1.5.3</version>
<version>${replacer.version}</version>
<executions>
<execution>
<id>replace-swagger-ui</id>
@ -123,22 +153,35 @@
<inherited>false</inherited>
<configuration>
<file>${project.build.directory}/swagger-ui.unpacked/META-INF/resources/webjars/swagger-ui/${swagger-ui.version}/index.html</file>
<replacements>
<replacement>
<token>Swagger UI</token>
<value>Qortal API Documentation</value>
</replacement>
</replacements>
</configuration>
</execution>
<execution>
<id>replace-swagger-ui-json</id>
<phase>generate-resources</phase>
<goals>
<goal>replace</goal>
</goals>
<inherited>false</inherited>
<configuration>
<file>${project.build.directory}/swagger-ui.unpacked/META-INF/resources/webjars/swagger-ui/${swagger-ui.version}/swagger-initializer.js</file>
<replacements>
<replacement>
<token>https://petstore.swagger.io/v2/swagger.json</token>
<value>/openapi.json</value>
</replacement>
<replacement>
<token>Swagger UI</token>
<value>API Documentation</value>
</replacement>
<replacement>
<token>deepLinking: true,</token>
<value>
deepLinking: true,
tagsSorter: "alpha",
operationsSorter:
"alpha",
operationsSorter: "alpha",
validatorUrl: false,
</value>
</replacement>
</replacements>
@ -154,11 +197,13 @@
<configuration>
<file>${project.build.outputDirectory}/git.properties</file>
<regex>true</regex>
<regexFlags><regexFlag>MULTILINE</regexFlag></regexFlags>
<regexFlags>
<regexFlag>MULTILINE</regexFlag>
</regexFlags>
<replacements>
<replacement>
<token>^(#.*$[\n\r]*)</token>
<value></value>
<value/>
</replacement>
</replacements>
</configuration>
@ -168,7 +213,10 @@
<!-- add swagger-ui as resource to output package -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.1.0</version>
<version>${maven-resources-plugin.version}</version>
<configuration>
<propertiesEncoding>ISO-8859-1</propertiesEncoding>
</configuration>
<executions>
<execution>
<id>copy-resources</id>
@ -192,7 +240,7 @@
<plugin>
<groupId>com.github.bohnman</groupId>
<artifactId>package-info-maven-plugin</artifactId>
<version>${package-info-maven-plugin.version}</version>
<version>${maven-package-info-plugin.version}</version>
<configuration>
<packages>
<package>
@ -222,7 +270,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.0.0</version>
<version>${maven-build-helper-plugin.version}</version>
<executions>
<execution>
<phase>generate-sources</phase>
@ -240,7 +288,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.2.0</version>
<version>${maven-jar-plugin.version}</version>
<configuration>
<archive>
<manifest>
@ -258,13 +306,12 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<version>${maven-shade-plugin.version}</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
<artifactSet>
<excludes>
<!-- Don't include original swagger-UI as we're including our own
modified version -->
<!-- Don't include original swagger-UI as we're including our own modified version -->
<exclude>org.webjars:swagger-ui</exclude>
<!-- Don't include JUnit as it's for testing only! -->
<exclude>junit:junit</exclude>
@ -296,6 +343,7 @@
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.qortal.controller.Controller</mainClass>
<manifestEntries>
<Multi-Release>true</Multi-Release>
<Class-Path>. ..</Class-Path>
</manifestEntries>
</transformer>
@ -307,7 +355,7 @@
<plugin>
<groupId>io.github.zlika</groupId>
<artifactId>reproducible-build-maven-plugin</artifactId>
<version>0.11</version>
<version>${maven-reproducible-build-plugin.version}</version>
<executions>
<execution>
<phase>package</phase>
@ -324,7 +372,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version>
<version>${maven-surefire-plugin.version}</version>
<configuration>
<skipTests>${skipTests}</skipTests>
</configuration>
@ -336,46 +384,34 @@
<plugin>
<groupId>org.eclipse.m2e</groupId>
<artifactId>lifecycle-mapping</artifactId>
<version>1.0.0</version>
<version>${lifecycle-mapping.version}</version>
<configuration>
<lifecycleMappingMetadata>
<pluginExecutions>
<pluginExecution>
<pluginExecutionFilter>
<groupId>
org.apache.maven.plugins
</groupId>
<artifactId>
maven-dependency-plugin
</artifactId>
<versionRange>
[2.8,)
</versionRange>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>${maven-dependency-plugin.version}</version>
<goals>
<goal>unpack</goal>
</goals>
</pluginExecutionFilter>
<action>
<execute></execute>
<execute/>
</action>
</pluginExecution>
<pluginExecution>
<pluginExecutionFilter>
<groupId>
com.google.code.maven-replacer-plugin
</groupId>
<artifactId>
replacer
</artifactId>
<versionRange>
[1.5.3,)
</versionRange>
<groupId>com.google.code.maven-replacer-plugin</groupId>
<artifactId>replacer</artifactId>
<version>${replacer.version}</version>
<goals>
<goal>replace</goal>
</goals>
</pluginExecutionFilter>
<action>
<execute></execute>
<execute/>
</action>
</pluginExecution>
</pluginExecutions>
@ -402,15 +438,17 @@
<dependency>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.0.0</version>
<scope>provided</scope><!-- needed for build, not for runtime -->
<version>${maven-build-helper-plugin.version}</version>
<scope>provided</scope>
<!-- needed for build, not for runtime -->
</dependency>
<!-- https://mvnrepository.com/artifact/com.github.bohnman/package-info-maven-plugin -->
<dependency>
<groupId>com.github.bohnman</groupId>
<artifactId>package-info-maven-plugin</artifactId>
<version>${package-info-maven-plugin.version}</version>
<scope>provided</scope><!-- needed for build, not for runtime -->
<version>${maven-package-info-plugin.version}</version>
<scope>provided</scope>
<!-- needed for build, not for runtime -->
</dependency>
<!-- HSQLDB for repository -->
<dependency>
@ -424,6 +462,12 @@
<artifactId>AT</artifactId>
<version>${ciyam-at.version}</version>
</dependency>
<!-- UPnP support -->
<dependency>
<groupId>com.dosse</groupId>
<artifactId>WaifUPnP</artifactId>
<version>${upnp.version}</version>
</dependency>
<!-- Bitcoin support -->
<dependency>
<groupId>org.bitcoinj</groupId>
@ -432,7 +476,7 @@
</dependency>
<!-- For Litecoin, etc. support, requires bitcoinj -->
<dependency>
<groupId>com.github.jjos2372</groupId>
<groupId>com.github.qortal</groupId>
<artifactId>altcoinj</artifactId>
<version>${altcoinj.version}</version>
</dependency>
@ -440,12 +484,12 @@
<dependency>
<groupId>com.googlecode.json-simple</groupId>
<artifactId>json-simple</artifactId>
<version>1.1.1</version>
<version>${json-simple.version}</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20210307</version>
<version>${json.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@ -462,6 +506,11 @@
<artifactId>commons-compress</artifactId>
<version>${commons-compress.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
</dependency>
<dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
@ -471,7 +520,7 @@
<dependency>
<groupId>io.druid</groupId>
<artifactId>extendedset</artifactId>
<version>0.12.3</version>
<version>${extendedset.version}</version>
<exclusions>
<!-- exclude old versions of jackson-annotations / jackson-core -->
<exclusion>
@ -542,18 +591,29 @@
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>4.0.1</version>
<version>${javax.servlet-api.version}</version>
</dependency>
<dependency>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
<version>1.5.0-b01</version>
<version>${mail.version}</version>
</dependency>
<!-- Unicode homoglyph utilities -->
<dependency>
<groupId>net.codebox</groupId>
<artifactId>homoglyph</artifactId>
<version>1.2.0</version>
<version>${homoglyph.version}</version>
</dependency>
<!-- Unicode support -->
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId>
<version>${icu4j.version}</version>
</dependency>
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j-charset</artifactId>
<version>${icu4j.version}</version>
</dependency>
<!-- Jetty -->
<dependency>
@ -605,7 +665,8 @@
<artifactId>jersey-hk2</artifactId>
<version>${jersey.version}</version>
<exclusions>
<exclusion><!-- exclude javax.inject-1.jar because other jersey modules include javax.inject v2+ -->
<exclusion>
<!-- exclude javax.inject-1.jar because other jersey modules include javax.inject v2+ -->
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
</exclusion>
@ -632,7 +693,8 @@
<artifactId>swagger-jaxrs2-servlet-initializer</artifactId>
<version>${swagger-api.version}</version>
<exclusions>
<exclusion><!-- excluded because included in swagger-jaxrs2-servlet-initializer -->
<exclusion>
<!-- excluded because included in swagger-jaxrs2-servlet-initializer -->
<groupId>io.swagger.core.v3</groupId>
<artifactId>swagger-integration</artifactId>
</exclusion>
@ -648,12 +710,12 @@
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.3.1</version>
<version>${junit-jupiter-engine.version}</version>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
<version>1.3</version>
<version>${hamcrest-library.version}</version>
</dependency>
-->
<!-- BouncyCastle for crypto, including TLS secure networking -->
@ -667,5 +729,46 @@
<artifactId>bctls-jdk15on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>${jsoup.version}</version>
</dependency>
<dependency>
<groupId>io.github.java-diff-utils</groupId>
<artifactId>java-diff-utils</artifactId>
<version>${java-diff-utils.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>com.j256.simplemagic</groupId>
<artifactId>simplemagic</artifactId>
<version>${simplemagic.version}</version>
</dependency>
<!-- JAXB runtime for WADL support -->
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
<version>${jaxb-runtime.version}</version>
</dependency>
</dependencies>
</project>

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,100 @@
/*
* Copyright (C) 2009 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* LiteWalletJni code based on https://github.com/PirateNetwork/cordova-plugin-litewallet
*
* MIT License
*
* Copyright (c) 2020 Zero Currency Coin
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package com.rust.litewalletjni;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.PirateChainWalletController;
import java.nio.file.Path;
import java.nio.file.Paths;
public class LiteWalletJni {
protected static final Logger LOGGER = LogManager.getLogger(LiteWalletJni.class);
public static native String initlogging();
public static native String initnew(final String serveruri, final String params, final String saplingOutputb64, final String saplingSpendb64);
public static native String initfromseed(final String serveruri, final String params, final String seed, final String birthday, final String saplingOutputb64, final String saplingSpendb64);
public static native String initfromb64(final String serveruri, final String params, final String datab64, final String saplingOutputb64, final String saplingSpendb64);
public static native String save();
public static native String execute(final String cmd, final String args);
public static native String getseedphrase();
public static native String getseedphrasefromentropyb64(final String entropy64);
public static native String checkseedphrase(final String input);
private static boolean loaded = false;
public static void loadLibrary() {
if (loaded) {
return;
}
String osName = System.getProperty("os.name");
String osArchitecture = System.getProperty("os.arch");
LOGGER.info("OS Name: {}", osName);
LOGGER.info("OS Architecture: {}", osArchitecture);
try {
String libFileName = PirateChainWalletController.getRustLibFilename();
if (libFileName == null) {
LOGGER.info("Library not found for OS: {}, arch: {}", osName, osArchitecture);
return;
}
Path libPath = Paths.get(PirateChainWalletController.getRustLibOuterDirectory().toString(), libFileName);
System.load(libPath.toAbsolutePath().toString());
loaded = true;
}
catch (UnsatisfiedLinkError e) {
LOGGER.info("Unable to load library");
}
}
public static boolean isLoaded() {
return loaded;
}
}

View File

@ -1,10 +1,10 @@
package org.hsqldb.jdbc;
import org.hsqldb.jdbc.pool.JDBCPooledConnection;
import java.sql.Connection;
import java.sql.SQLException;
import org.hsqldb.jdbc.pool.JDBCPooledConnection;
public class HSQLDBPool extends JDBCPool {
public HSQLDBPool(int poolSize) {

View File

@ -0,0 +1,227 @@
package org.qortal;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.qortal.api.ApiKey;
import org.qortal.api.ApiRequest;
import org.qortal.controller.BootstrapNode;
import org.qortal.settings.Settings;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.Security;
import java.util.*;
import java.util.stream.Collectors;
import static org.qortal.controller.BootstrapNode.AGENTLIB_JVM_HOLDER_ARG;
public class ApplyBootstrap {
static {
// This static block will be called before others if using ApplyBootstrap.main()
// Log into different files for bootstrap - this has to be before LogManger.getLogger() calls
System.setProperty("log4j2.filenameTemplate", "log-apply-bootstrap.txt");
// This must go before any calls to LogManager/Logger
System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
}
private static final Logger LOGGER = LogManager.getLogger(ApplyBootstrap.class);
private static final String JAR_FILENAME = BootstrapNode.JAR_FILENAME;
private static final String WINDOWS_EXE_LAUNCHER = "qortal.exe";
private static final String JAVA_TOOL_OPTIONS_NAME = "JAVA_TOOL_OPTIONS";
private static final String JAVA_TOOL_OPTIONS_VALUE = "";
private static final long CHECK_INTERVAL = 15 * 1000L; // ms
private static final int MAX_ATTEMPTS = 20;
public static void main(String[] args) {
Security.insertProviderAt(new BouncyCastleProvider(), 0);
Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
// Load/check settings, which potentially sets up blockchain config, etc.
if (args.length > 0)
Settings.fileInstance(args[0]);
else
Settings.getInstance();
LOGGER.info("Applying bootstrap...");
// Shutdown node using API
if (!shutdownNode())
return;
// Delete db
deleteDB();
// Restart node
restartNode(args);
LOGGER.info("Bootstrapping...");
}
private static boolean shutdownNode() {
String baseUri = "http://localhost:" + Settings.getInstance().getApiPort() + "/";
LOGGER.info(() -> String.format("Shutting down node using API via %s", baseUri));
// The /admin/stop endpoint requires an API key, which may or may not be already generated
boolean apiKeyNewlyGenerated = false;
ApiKey apiKey = null;
try {
apiKey = new ApiKey();
if (!apiKey.generated()) {
apiKey.generate();
apiKeyNewlyGenerated = true;
LOGGER.info("Generated API key");
}
} catch (IOException e) {
LOGGER.info("Error loading API key: {}", e.getMessage());
}
// Create GET params
Map<String, String> params = new HashMap<>();
if (apiKey != null) {
params.put("apiKey", apiKey.toString());
}
// Attempt to stop the node
int attempt;
for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) {
final int attemptForLogging = attempt;
LOGGER.info(() -> String.format("Attempt #%d out of %d to shutdown node", attemptForLogging + 1, MAX_ATTEMPTS));
String response = ApiRequest.perform(baseUri + "admin/stop", params);
if (response == null) {
// No response - consider node shut down
if (apiKeyNewlyGenerated) {
// API key was newly generated for bootstrapping node, so we need to remove it
ApplyBootstrap.removeGeneratedApiKey();
}
return true;
}
LOGGER.info(() -> String.format("Response from API: %s", response));
try {
Thread.sleep(CHECK_INTERVAL);
} catch (InterruptedException e) {
// We still need to check...
break;
}
}
if (apiKeyNewlyGenerated) {
// API key was newly generated for bootstrapping node, so we need to remove it
ApplyBootstrap.removeGeneratedApiKey();
}
if (attempt == MAX_ATTEMPTS) {
LOGGER.error("Failed to shutdown node - giving up");
return false;
}
return true;
}
private static void removeGeneratedApiKey() {
try {
LOGGER.info("Removing newly generated API key...");
// Delete the API key since it was only generated for bootstrapping node
ApiKey apiKey = new ApiKey();
apiKey.delete();
} catch (IOException e) {
LOGGER.info("Error loading or deleting API key: {}", e.getMessage());
}
}
private static void deleteDB() {
// Get the repository path from settings
String repositoryPath = Settings.getInstance().getRepositoryPath();
LOGGER.debug(String.format("Repository path: %s", repositoryPath));
try {
Path directory = Paths.get(repositoryPath);
Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
LOGGER.error("Error deleting DB: {}", e.getMessage());
}
}
private static void restartNode(String[] args) {
String javaHome = System.getProperty("java.home");
LOGGER.debug(() -> String.format("Java home: %s", javaHome));
Path javaBinary = Paths.get(javaHome, "bin", "java");
LOGGER.debug(() -> String.format("Java binary: %s", javaBinary));
Path exeLauncher = Paths.get(WINDOWS_EXE_LAUNCHER);
LOGGER.debug(() -> String.format("Windows EXE launcher: %s", exeLauncher));
List<String> javaCmd;
if (Files.exists(exeLauncher)) {
javaCmd = Arrays.asList(exeLauncher.toString());
} else {
javaCmd = new ArrayList<>();
// Java runtime binary itself
javaCmd.add(javaBinary.toString());
// JVM arguments
javaCmd.addAll(ManagementFactory.getRuntimeMXBean().getInputArguments());
// Reapply any retained, but disabled, -agentlib JVM arg
javaCmd = javaCmd.stream()
.map(arg -> arg.replace(AGENTLIB_JVM_HOLDER_ARG, "-agentlib"))
.collect(Collectors.toList());
// Call mainClass in JAR
javaCmd.addAll(Arrays.asList("-jar", JAR_FILENAME));
// Add saved command-line args
javaCmd.addAll(Arrays.asList(args));
}
try {
LOGGER.info(String.format("Restarting node with: %s", String.join(" ", javaCmd)));
ProcessBuilder processBuilder = new ProcessBuilder(javaCmd);
if (Files.exists(exeLauncher)) {
LOGGER.debug(() -> String.format("Setting env %s to %s", JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE));
processBuilder.environment().put(JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE);
}
// New process will inherit our stdout and stderr
processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT);
Process process = processBuilder.start();
// Nothing to pipe to new process, so close output stream (process's stdin)
process.getOutputStream().close();
} catch (Exception e) {
LOGGER.error(String.format("Failed to restart node (BAD): %s", e.getMessage()));
}
}
}

View File

@ -0,0 +1,196 @@
package org.qortal;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.qortal.api.ApiKey;
import org.qortal.api.ApiRequest;
import org.qortal.controller.RestartNode;
import org.qortal.settings.Settings;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.Security;
import java.util.*;
import java.util.stream.Collectors;
import static org.qortal.controller.RestartNode.AGENTLIB_JVM_HOLDER_ARG;
public class ApplyRestart {
static {
// This static block will be called before others if using ApplyRestart.main()
// Log into different files for restart node - this has to be before LogManger.getLogger() calls
System.setProperty("log4j2.filenameTemplate", "log-apply-restart.txt");
// This must go before any calls to LogManager/Logger
System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
}
private static final Logger LOGGER = LogManager.getLogger(ApplyRestart.class);
private static final String JAR_FILENAME = RestartNode.JAR_FILENAME;
private static final String WINDOWS_EXE_LAUNCHER = "qortal.exe";
private static final String JAVA_TOOL_OPTIONS_NAME = "JAVA_TOOL_OPTIONS";
private static final String JAVA_TOOL_OPTIONS_VALUE = "";
private static final long CHECK_INTERVAL = 10 * 1000L; // ms
private static final int MAX_ATTEMPTS = 12;
public static void main(String[] args) {
Security.insertProviderAt(new BouncyCastleProvider(), 0);
Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
// Load/check settings, which potentially sets up blockchain config, etc.
if (args.length > 0)
Settings.fileInstance(args[0]);
else
Settings.getInstance();
LOGGER.info("Applying restart...");
// Shutdown node using API
if (!shutdownNode())
return;
// Restart node
restartNode(args);
LOGGER.info("Restarting...");
}
private static boolean shutdownNode() {
String baseUri = "http://localhost:" + Settings.getInstance().getApiPort() + "/";
LOGGER.info(() -> String.format("Shutting down node using API via %s", baseUri));
// The /admin/stop endpoint requires an API key, which may or may not be already generated
boolean apiKeyNewlyGenerated = false;
ApiKey apiKey = null;
try {
apiKey = new ApiKey();
if (!apiKey.generated()) {
apiKey.generate();
apiKeyNewlyGenerated = true;
LOGGER.info("Generated API key");
}
} catch (IOException e) {
LOGGER.info("Error loading API key: {}", e.getMessage());
}
// Create GET params
Map<String, String> params = new HashMap<>();
if (apiKey != null) {
params.put("apiKey", apiKey.toString());
}
// Attempt to stop the node
int attempt;
for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) {
final int attemptForLogging = attempt;
LOGGER.info(() -> String.format("Attempt #%d out of %d to shutdown node", attemptForLogging + 1, MAX_ATTEMPTS));
String response = ApiRequest.perform(baseUri + "admin/stop", params);
if (response == null) {
// No response - consider node shut down
if (apiKeyNewlyGenerated) {
// API key was newly generated for restarting node, so we need to remove it
ApplyRestart.removeGeneratedApiKey();
}
return true;
}
LOGGER.info(() -> String.format("Response from API: %s", response));
try {
Thread.sleep(CHECK_INTERVAL);
} catch (InterruptedException e) {
// We still need to check...
break;
}
}
if (apiKeyNewlyGenerated) {
// API key was newly generated for restarting node, so we need to remove it
ApplyRestart.removeGeneratedApiKey();
}
if (attempt == MAX_ATTEMPTS) {
LOGGER.error("Failed to shutdown node - giving up");
return false;
}
return true;
}
private static void removeGeneratedApiKey() {
try {
LOGGER.info("Removing newly generated API key...");
// Delete the API key since it was only generated for restarting node
ApiKey apiKey = new ApiKey();
apiKey.delete();
} catch (IOException e) {
LOGGER.info("Error loading or deleting API key: {}", e.getMessage());
}
}
private static void restartNode(String[] args) {
String javaHome = System.getProperty("java.home");
LOGGER.debug(() -> String.format("Java home: %s", javaHome));
Path javaBinary = Paths.get(javaHome, "bin", "java");
LOGGER.debug(() -> String.format("Java binary: %s", javaBinary));
Path exeLauncher = Paths.get(WINDOWS_EXE_LAUNCHER);
LOGGER.debug(() -> String.format("Windows EXE launcher: %s", exeLauncher));
List<String> javaCmd;
if (Files.exists(exeLauncher)) {
javaCmd = Arrays.asList(exeLauncher.toString());
} else {
javaCmd = new ArrayList<>();
// Java runtime binary itself
javaCmd.add(javaBinary.toString());
// JVM arguments
javaCmd.addAll(ManagementFactory.getRuntimeMXBean().getInputArguments());
// Reapply any retained, but disabled, -agentlib JVM arg
javaCmd = javaCmd.stream()
.map(arg -> arg.replace(AGENTLIB_JVM_HOLDER_ARG, "-agentlib"))
.collect(Collectors.toList());
// Call mainClass in JAR
javaCmd.addAll(Arrays.asList("-jar", JAR_FILENAME));
// Add saved command-line args
javaCmd.addAll(Arrays.asList(args));
}
try {
LOGGER.debug(String.format("Restarting node with: %s", String.join(" ", javaCmd)));
ProcessBuilder processBuilder = new ProcessBuilder(javaCmd);
if (Files.exists(exeLauncher)) {
LOGGER.debug(() -> String.format("Setting env %s to %s", JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE));
processBuilder.environment().put(JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE);
}
// New process will inherit our stdout and stderr
processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT);
Process process = processBuilder.start();
// Nothing to pipe to new process, so close output stream (process's stdin)
process.getOutputStream().close();
} catch (Exception e) {
LOGGER.error(String.format("Failed to restart node (BAD): %s", e.getMessage()));
}
}
}

View File

@ -1,5 +1,14 @@
package org.qortal;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.qortal.api.ApiKey;
import org.qortal.api.ApiRequest;
import org.qortal.controller.AutoUpdate;
import org.qortal.settings.Settings;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.nio.file.Files;
@ -7,17 +16,10 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.Security;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.qortal.api.ApiRequest;
import org.qortal.controller.AutoUpdate;
import org.qortal.settings.Settings;
import static org.qortal.controller.AutoUpdate.AGENTLIB_JVM_HOLDER_ARG;
public class ApplyUpdate {
@ -36,9 +38,9 @@ public class ApplyUpdate {
private static final String NEW_JAR_FILENAME = AutoUpdate.NEW_JAR_FILENAME;
private static final String WINDOWS_EXE_LAUNCHER = "qortal.exe";
private static final String JAVA_TOOL_OPTIONS_NAME = "JAVA_TOOL_OPTIONS";
private static final String JAVA_TOOL_OPTIONS_VALUE = "-XX:MaxRAMFraction=4";
private static final String JAVA_TOOL_OPTIONS_VALUE = "";
private static final long CHECK_INTERVAL = 10 * 1000L; // ms
private static final long CHECK_INTERVAL = 30 * 1000L; // ms
private static final int MAX_ATTEMPTS = 12;
public static void main(String[] args) {
@ -70,14 +72,40 @@ public class ApplyUpdate {
String baseUri = "http://localhost:" + Settings.getInstance().getApiPort() + "/";
LOGGER.info(() -> String.format("Shutting down node using API via %s", baseUri));
// The /admin/stop endpoint requires an API key, which may or may not be already generated
boolean apiKeyNewlyGenerated = false;
ApiKey apiKey = null;
try {
apiKey = new ApiKey();
if (!apiKey.generated()) {
apiKey.generate();
apiKeyNewlyGenerated = true;
LOGGER.info("Generated API key");
}
} catch (IOException e) {
LOGGER.info("Error loading API key: {}", e.getMessage());
}
// Create GET params
Map<String, String> params = new HashMap<>();
if (apiKey != null) {
params.put("apiKey", apiKey.toString());
}
// Attempt to stop the node
int attempt;
for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) {
final int attemptForLogging = attempt;
LOGGER.info(() -> String.format("Attempt #%d out of %d to shutdown node", attemptForLogging + 1, MAX_ATTEMPTS));
String response = ApiRequest.perform(baseUri + "admin/stop", null);
if (response == null)
String response = ApiRequest.perform(baseUri + "admin/stop", params);
if (response == null) {
// No response - consider node shut down
if (apiKeyNewlyGenerated) {
// API key was newly generated for this auto update, so we need to remove it
ApplyUpdate.removeGeneratedApiKey();
}
return true;
}
LOGGER.info(() -> String.format("Response from API: %s", response));
@ -89,6 +117,11 @@ public class ApplyUpdate {
}
}
if (apiKeyNewlyGenerated) {
// API key was newly generated for this auto update, so we need to remove it
ApplyUpdate.removeGeneratedApiKey();
}
if (attempt == MAX_ATTEMPTS) {
LOGGER.error("Failed to shutdown node - giving up");
return false;
@ -97,6 +130,19 @@ public class ApplyUpdate {
return true;
}
private static void removeGeneratedApiKey() {
try {
LOGGER.info("Removing newly generated API key...");
// Delete the API key since it was only generated for this auto update
ApiKey apiKey = new ApiKey();
apiKey.delete();
} catch (IOException e) {
LOGGER.error("Error loading or deleting API key: {}", e.getMessage());
}
}
private static void replaceJar() {
// Assuming current working directory contains the JAR files
Path realJar = Paths.get(JAR_FILENAME);
@ -135,13 +181,13 @@ public class ApplyUpdate {
private static void restartNode(String[] args) {
String javaHome = System.getProperty("java.home");
LOGGER.info(() -> String.format("Java home: %s", javaHome));
LOGGER.debug(() -> String.format("Java home: %s", javaHome));
Path javaBinary = Paths.get(javaHome, "bin", "java");
LOGGER.info(() -> String.format("Java binary: %s", javaBinary));
LOGGER.debug(() -> String.format("Java binary: %s", javaBinary));
Path exeLauncher = Paths.get(WINDOWS_EXE_LAUNCHER);
LOGGER.info(() -> String.format("Windows EXE launcher: %s", exeLauncher));
LOGGER.debug(() -> String.format("Windows EXE launcher: %s", exeLauncher));
List<String> javaCmd;
if (Files.exists(exeLauncher)) {
@ -154,6 +200,11 @@ public class ApplyUpdate {
// JVM arguments
javaCmd.addAll(ManagementFactory.getRuntimeMXBean().getInputArguments());
// Reapply any retained, but disabled, -agentlib JVM arg
javaCmd = javaCmd.stream()
.map(arg -> arg.replace(AGENTLIB_JVM_HOLDER_ARG, "-agentlib"))
.collect(Collectors.toList());
// Call mainClass in JAR
javaCmd.addAll(Arrays.asList("-jar", JAR_FILENAME));
@ -162,17 +213,24 @@ public class ApplyUpdate {
}
try {
LOGGER.info(() -> String.format("Restarting node with: %s", String.join(" ", javaCmd)));
LOGGER.debug(String.format("Restarting node with: %s", String.join(" ", javaCmd)));
ProcessBuilder processBuilder = new ProcessBuilder(javaCmd);
if (Files.exists(exeLauncher)) {
LOGGER.info(() -> String.format("Setting env %s to %s", JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE));
LOGGER.debug(() -> String.format("Setting env %s to %s", JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE));
processBuilder.environment().put(JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE);
}
processBuilder.start();
} catch (IOException e) {
// New process will inherit our stdout and stderr
processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT);
Process process = processBuilder.start();
// Nothing to pipe to new process, so close output stream (process's stdin)
process.getOutputStream().close();
} catch (Exception e) {
LOGGER.error(String.format("Failed to restart node (BAD): %s", e.getMessage()));
}
}

View File

@ -1,8 +1,5 @@
package org.qortal;
import java.security.Security;
import java.util.concurrent.TimeoutException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
@ -15,6 +12,9 @@ import org.qortal.repository.RepositoryManager;
import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory;
import org.qortal.settings.Settings;
import java.security.Security;
import java.util.concurrent.TimeoutException;
public class RepositoryMaintenance {
static {

View File

@ -1,5 +1,7 @@
package org.qortal;
import org.qortal.controller.AutoUpdate;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -7,8 +9,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.qortal.controller.AutoUpdate;
public class XorUpdate {
private static final byte XOR_VALUE = AutoUpdate.XOR_VALUE;

View File

@ -1,20 +1,22 @@
package org.qortal.account;
import static org.qortal.utils.Amounts.prettyAmount;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.block.BlockChain;
import org.qortal.controller.LiteNode;
import org.qortal.data.account.AccountBalanceData;
import org.qortal.data.account.AccountData;
import org.qortal.data.account.RewardShareData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import static org.qortal.utils.Amounts.prettyAmount;
@XmlAccessorType(XmlAccessType.NONE) // Stops JAX-RS errors when unmarshalling blockchain config
public class Account {
@ -59,7 +61,17 @@ public class Account {
// Balance manipulations - assetId is 0 for QORT
public long getConfirmedBalance(long assetId) throws DataException {
AccountBalanceData accountBalanceData = this.repository.getAccountRepository().getBalance(this.address, assetId);
AccountBalanceData accountBalanceData;
if (Settings.getInstance().isLite()) {
// Lite nodes request data from peers instead of the local db
accountBalanceData = LiteNode.getInstance().fetchAccountBalance(this.address, assetId);
}
else {
// All other node types fetch from the local db
accountBalanceData = this.repository.getAccountRepository().getBalance(this.address, assetId);
}
if (accountBalanceData == null)
return 0;
@ -199,12 +211,24 @@ public class Account {
if (level != null && level >= BlockChain.getInstance().getMinAccountLevelToMint())
return true;
if (Account.isFounder(accountData.getFlags()))
// Founders can always mint, unless they have a penalty
if (Account.isFounder(accountData.getFlags()) && accountData.getBlocksMintedPenalty() == 0)
return true;
return false;
}
/** Returns account's blockMinted (0+) or null if account not found in repository. */
public Integer getBlocksMinted() throws DataException {
return this.repository.getAccountRepository().getMintedBlockCount(this.address);
}
/** Returns account's blockMintedPenalty or null if account not found in repository. */
public Integer getBlocksMintedPenalty() throws DataException {
return this.repository.getAccountRepository().getBlocksMintedPenaltyCount(this.address);
}
/** Returns whether account can build reward-shares.
* <p>
* To be able to create reward-shares, the account needs to pass at least one of these tests:<br>
@ -225,7 +249,7 @@ public class Account {
if (level != null && level >= BlockChain.getInstance().getMinAccountLevelToRewardShare())
return true;
if (Account.isFounder(accountData.getFlags()))
if (Account.isFounder(accountData.getFlags()) && accountData.getBlocksMintedPenalty() == 0)
return true;
return false;
@ -253,7 +277,7 @@ public class Account {
/**
* Returns 'effective' minting level, or zero if account does not exist/cannot mint.
* <p>
* For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config.
* For founder accounts with no penalty, this returns "founderEffectiveMintingLevel" from blockchain config.
*
* @return 0+
* @throws DataException
@ -263,7 +287,8 @@ public class Account {
if (accountData == null)
return 0;
if (Account.isFounder(accountData.getFlags()))
// Founders are assigned a different effective minting level, as long as they have no penalty
if (Account.isFounder(accountData.getFlags()) && accountData.getBlocksMintedPenalty() == 0)
return BlockChain.getInstance().getFounderEffectiveMintingLevel();
return accountData.getLevel();
@ -271,8 +296,6 @@ public class Account {
/**
* Returns 'effective' minting level, or zero if reward-share does not exist.
* <p>
* For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config.
*
* @param repository
* @param rewardSharePublicKey
@ -288,5 +311,26 @@ public class Account {
Account rewardShareMinter = new Account(repository, rewardShareData.getMinter());
return rewardShareMinter.getEffectiveMintingLevel();
}
/**
* Returns 'effective' minting level, with a fix for the zero level.
* <p>
* For founder accounts with no penalty, this returns "founderEffectiveMintingLevel" from blockchain config.
*
* @param repository
* @param rewardSharePublicKey
* @return 0+
* @throws DataException
*/
public static int getRewardShareEffectiveMintingLevelIncludingLevelZero(Repository repository, byte[] rewardSharePublicKey) throws DataException {
// Find actual minter and get their effective minting level
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(rewardSharePublicKey);
if (rewardShareData == null)
return 0;
else if (!rewardShareData.getMinter().equals(rewardShareData.getRecipient())) // Sponsorship reward share
return 0;
Account rewardShareMinter = new Account(repository, rewardShareData.getMinter());
return rewardShareMinter.getEffectiveMintingLevel();
}
}

View File

@ -1,15 +1,15 @@
package org.qortal.account;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.function.BinaryOperator;
import org.qortal.data.account.AccountData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.utils.Pair;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.function.BinaryOperator;
/**
* Account lastReference caching
* <p>

View File

@ -11,15 +11,15 @@ public class PrivateKeyAccount extends PublicKeyAccount {
private final Ed25519PrivateKeyParameters edPrivateKeyParams;
/**
* Create PrivateKeyAccount using byte[32] seed.
* Create PrivateKeyAccount using byte[32] private key.
*
* @param seed
* @param privateKey
* byte[32] used to create private/public key pair
* @throws IllegalArgumentException
* if passed invalid seed
* if passed invalid privateKey
*/
public PrivateKeyAccount(Repository repository, byte[] seed) {
this(repository, new Ed25519PrivateKeyParameters(seed, 0));
public PrivateKeyAccount(Repository repository, byte[] privateKey) {
this(repository, new Ed25519PrivateKeyParameters(privateKey, 0));
}
private PrivateKeyAccount(Repository repository, Ed25519PrivateKeyParameters edPrivateKeyParams) {
@ -37,10 +37,6 @@ public class PrivateKeyAccount extends PublicKeyAccount {
return this.privateKey;
}
public static byte[] toPublicKey(byte[] seed) {
return new Ed25519PrivateKeyParameters(seed, 0).generatePublicKey().getEncoded();
}
public byte[] sign(byte[] message) {
return Crypto.sign(this.edPrivateKeyParams, message);
}

View File

@ -0,0 +1,366 @@
package org.qortal.account;
import org.qortal.api.resource.TransactionsResource;
import org.qortal.asset.Asset;
import org.qortal.data.account.AccountData;
import org.qortal.data.naming.NameData;
import org.qortal.data.transaction.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.transaction.Transaction.TransactionType;
import java.util.*;
import java.util.stream.Collectors;
public class SelfSponsorshipAlgoV1 {
private final Repository repository;
private final String address;
private final AccountData accountData;
private final long snapshotTimestamp;
private final boolean override;
private int registeredNameCount = 0;
private int suspiciousCount = 0;
private int suspiciousPercent = 0;
private int consolidationCount = 0;
private int bulkIssuanceCount = 0;
private int recentSponsorshipCount = 0;
private List<RewardShareTransactionData> sponsorshipRewardShares = new ArrayList<>();
private final Map<String, List<TransactionData>> paymentsByAddress = new HashMap<>();
private final Set<String> sponsees = new LinkedHashSet<>();
private Set<String> consolidatedAddresses = new LinkedHashSet<>();
private final Set<String> zeroTransactionAddreses = new LinkedHashSet<>();
private final Set<String> penaltyAddresses = new LinkedHashSet<>();
public SelfSponsorshipAlgoV1(Repository repository, String address, long snapshotTimestamp, boolean override) throws DataException {
this.repository = repository;
this.address = address;
this.accountData = this.repository.getAccountRepository().getAccount(this.address);
this.snapshotTimestamp = snapshotTimestamp;
this.override = override;
}
public String getAddress() {
return this.address;
}
public Set<String> getPenaltyAddresses() {
return this.penaltyAddresses;
}
public void run() throws DataException {
if (this.accountData == null) {
// Nothing to do
return;
}
this.fetchSponsorshipRewardShares();
if (this.sponsorshipRewardShares.isEmpty()) {
// Nothing to do
return;
}
this.findConsolidatedRewards();
this.findBulkIssuance();
this.findRegisteredNameCount();
this.findRecentSponsorshipCount();
int score = this.calculateScore();
if (score <= 0 && !override) {
return;
}
String newAddress = this.getDestinationAccount(this.address);
while (newAddress != null) {
// Found destination account
this.penaltyAddresses.add(newAddress);
// Run algo for this address, but in "override" mode because it has already been flagged
SelfSponsorshipAlgoV1 algoV1 = new SelfSponsorshipAlgoV1(this.repository, newAddress, this.snapshotTimestamp, true);
algoV1.run();
this.penaltyAddresses.addAll(algoV1.getPenaltyAddresses());
newAddress = this.getDestinationAccount(newAddress);
}
this.penaltyAddresses.add(this.address);
if (this.override || this.recentSponsorshipCount < 20) {
this.penaltyAddresses.addAll(this.consolidatedAddresses);
this.penaltyAddresses.addAll(this.zeroTransactionAddreses);
}
else {
this.penaltyAddresses.addAll(this.sponsees);
}
}
private String getDestinationAccount(String address) throws DataException {
List<TransactionData> transferPrivsTransactions = fetchTransferPrivsForAddress(address);
if (transferPrivsTransactions.isEmpty()) {
// No TRANSFER_PRIVS transactions for this address
return null;
}
AccountData accountData = this.repository.getAccountRepository().getAccount(address);
if (accountData == null) {
return null;
}
for (TransactionData transactionData : transferPrivsTransactions) {
TransferPrivsTransactionData transferPrivsTransactionData = (TransferPrivsTransactionData) transactionData;
if (Arrays.equals(transferPrivsTransactionData.getSenderPublicKey(), accountData.getPublicKey())) {
return transferPrivsTransactionData.getRecipient();
}
}
return null;
}
private void findConsolidatedRewards() throws DataException {
List<String> sponseesThatSentRewards = new ArrayList<>();
Map<String, Integer> paymentRecipients = new HashMap<>();
// Collect outgoing payments of each sponsee
for (String sponseeAddress : this.sponsees) {
// Firstly fetch all payments for address, since the functions below depend on this data
this.fetchPaymentsForAddress(sponseeAddress);
// Check if the address has zero relevant transactions
if (this.hasZeroTransactions(sponseeAddress)) {
this.zeroTransactionAddreses.add(sponseeAddress);
}
// Get payment recipients
List<String> allPaymentRecipients = this.fetchOutgoingPaymentRecipientsForAddress(sponseeAddress);
if (allPaymentRecipients.isEmpty()) {
continue;
}
sponseesThatSentRewards.add(sponseeAddress);
List<String> addressesPaidByThisSponsee = new ArrayList<>();
for (String paymentRecipient : allPaymentRecipients) {
if (addressesPaidByThisSponsee.contains(paymentRecipient)) {
// We already tracked this association - don't allow multiple to stack up
continue;
}
addressesPaidByThisSponsee.add(paymentRecipient);
// Increment count for this recipient, or initialize to 1 if not present
if (paymentRecipients.computeIfPresent(paymentRecipient, (k, v) -> v + 1) == null) {
paymentRecipients.put(paymentRecipient, 1);
}
}
}
// Exclude addresses with a low number of payments
Map<String, Integer> filteredPaymentRecipients = paymentRecipients.entrySet().stream()
.filter(p -> p.getValue() != null && p.getValue() >= 10)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
// Now check how many sponsees have sent to this subset of addresses
Map<String, Integer> sponseesThatConsolidatedRewards = new HashMap<>();
for (String sponseeAddress : sponseesThatSentRewards) {
List<String> allPaymentRecipients = this.fetchOutgoingPaymentRecipientsForAddress(sponseeAddress);
// Remove any that aren't to one of the flagged recipients (i.e. consolidation)
allPaymentRecipients.removeIf(r -> !filteredPaymentRecipients.containsKey(r));
int count = allPaymentRecipients.size();
if (count == 0) {
continue;
}
if (sponseesThatConsolidatedRewards.computeIfPresent(sponseeAddress, (k, v) -> v + count) == null) {
sponseesThatConsolidatedRewards.put(sponseeAddress, count);
}
}
// Remove sponsees that have only sent a low number of payments to the filtered addresses
Map<String, Integer> filteredSponseesThatConsolidatedRewards = sponseesThatConsolidatedRewards.entrySet().stream()
.filter(p -> p.getValue() != null && p.getValue() >= 2)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
this.consolidationCount = sponseesThatConsolidatedRewards.size();
this.consolidatedAddresses = new LinkedHashSet<>(filteredSponseesThatConsolidatedRewards.keySet());
this.suspiciousCount = this.consolidationCount + this.zeroTransactionAddreses.size();
this.suspiciousPercent = (int)(this.suspiciousCount / (float) this.sponsees.size() * 100);
}
private void findBulkIssuance() {
Long lastTimestamp = null;
for (RewardShareTransactionData rewardShareTransactionData : sponsorshipRewardShares) {
long timestamp = rewardShareTransactionData.getTimestamp();
if (timestamp >= this.snapshotTimestamp) {
continue;
}
if (lastTimestamp != null) {
if (timestamp - lastTimestamp < 3*60*1000L) {
this.bulkIssuanceCount++;
}
}
lastTimestamp = timestamp;
}
}
private void findRegisteredNameCount() throws DataException {
int registeredNameCount = 0;
for (String sponseeAddress : sponsees) {
List<NameData> names = repository.getNameRepository().getNamesByOwner(sponseeAddress);
for (NameData name : names) {
if (name.getRegistered() < this.snapshotTimestamp) {
registeredNameCount++;
break;
}
}
}
this.registeredNameCount = registeredNameCount;
}
private void findRecentSponsorshipCount() {
final long referenceTimestamp = this.snapshotTimestamp - (365 * 24 * 60 * 60 * 1000L);
int recentSponsorshipCount = 0;
for (RewardShareTransactionData rewardShare : sponsorshipRewardShares) {
if (rewardShare.getTimestamp() >= referenceTimestamp) {
recentSponsorshipCount++;
}
}
this.recentSponsorshipCount = recentSponsorshipCount;
}
private int calculateScore() {
final int suspiciousMultiplier = (this.suspiciousCount >= 100) ? this.suspiciousPercent : 1;
final int nameMultiplier = (this.sponsees.size() >= 50 && this.registeredNameCount == 0) ? 2 : 1;
final int consolidationMultiplier = Math.max(this.consolidationCount, 1);
final int bulkIssuanceMultiplier = Math.max(this.bulkIssuanceCount / 2, 1);
final int offset = 9;
return suspiciousMultiplier * nameMultiplier * consolidationMultiplier * bulkIssuanceMultiplier - offset;
}
private void fetchSponsorshipRewardShares() throws DataException {
List<RewardShareTransactionData> sponsorshipRewardShares = new ArrayList<>();
// Define relevant transactions
List<TransactionType> txTypes = List.of(TransactionType.REWARD_SHARE);
List<TransactionData> transactionDataList = fetchTransactions(repository, txTypes, this.address, false);
for (TransactionData transactionData : transactionDataList) {
if (transactionData.getType() != TransactionType.REWARD_SHARE) {
continue;
}
RewardShareTransactionData rewardShareTransactionData = (RewardShareTransactionData) transactionData;
// Skip removals
if (rewardShareTransactionData.getSharePercent() < 0) {
continue;
}
// Skip if not sponsored by this account
if (!Arrays.equals(rewardShareTransactionData.getCreatorPublicKey(), accountData.getPublicKey())) {
continue;
}
// Skip self shares
if (Objects.equals(rewardShareTransactionData.getRecipient(), this.address)) {
continue;
}
boolean duplicateFound = false;
for (RewardShareTransactionData existingRewardShare : sponsorshipRewardShares) {
if (Objects.equals(existingRewardShare.getRecipient(), rewardShareTransactionData.getRecipient())) {
// Duplicate
duplicateFound = true;
break;
}
}
if (!duplicateFound) {
sponsorshipRewardShares.add(rewardShareTransactionData);
this.sponsees.add(rewardShareTransactionData.getRecipient());
}
}
this.sponsorshipRewardShares = sponsorshipRewardShares;
}
private List<TransactionData> fetchTransferPrivsForAddress(String address) throws DataException {
return fetchTransactions(repository,
List.of(TransactionType.TRANSFER_PRIVS),
address, true);
}
private void fetchPaymentsForAddress(String address) throws DataException {
List<TransactionData> payments = fetchTransactions(repository,
Arrays.asList(TransactionType.PAYMENT, TransactionType.TRANSFER_ASSET),
address, false);
this.paymentsByAddress.put(address, payments);
}
private List<String> fetchOutgoingPaymentRecipientsForAddress(String address) {
List<String> outgoingPaymentRecipients = new ArrayList<>();
List<TransactionData> transactionDataList = this.paymentsByAddress.get(address);
if (transactionDataList == null) transactionDataList = new ArrayList<>();
transactionDataList.removeIf(t -> t.getTimestamp() >= this.snapshotTimestamp);
for (TransactionData transactionData : transactionDataList) {
switch (transactionData.getType()) {
case PAYMENT:
PaymentTransactionData paymentTransactionData = (PaymentTransactionData) transactionData;
if (!Objects.equals(paymentTransactionData.getRecipient(), address)) {
// Outgoing payment from this account
outgoingPaymentRecipients.add(paymentTransactionData.getRecipient());
}
break;
case TRANSFER_ASSET:
TransferAssetTransactionData transferAssetTransactionData = (TransferAssetTransactionData) transactionData;
if (transferAssetTransactionData.getAssetId() == Asset.QORT) {
if (!Objects.equals(transferAssetTransactionData.getRecipient(), address)) {
// Outgoing payment from this account
outgoingPaymentRecipients.add(transferAssetTransactionData.getRecipient());
}
}
break;
default:
break;
}
}
return outgoingPaymentRecipients;
}
private boolean hasZeroTransactions(String address) {
List<TransactionData> transactionDataList = this.paymentsByAddress.get(address);
if (transactionDataList == null) {
return true;
}
transactionDataList.removeIf(t -> t.getTimestamp() >= this.snapshotTimestamp);
return transactionDataList.isEmpty();
}
private static List<TransactionData> fetchTransactions(Repository repository, List<TransactionType> txTypes, String address, boolean reverse) throws DataException {
// Fetch all relevant transactions for this account
List<byte[]> signatures = repository.getTransactionRepository()
.getSignaturesMatchingCriteria(null, null, null, txTypes,
null, null, address, TransactionsResource.ConfirmationStatus.CONFIRMED,
null, null, reverse);
List<TransactionData> transactionDataList = new ArrayList<>();
for (byte[] signature : signatures) {
// Fetch transaction data
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (transactionData == null) {
continue;
}
transactionDataList.add(transactionData);
}
return transactionDataList;
}
}

View File

@ -0,0 +1,249 @@
package org.qortal.account;
import org.qortal.api.resource.TransactionsResource;
import org.qortal.asset.Asset;
import org.qortal.block.BlockChain;
import org.qortal.data.account.AccountData;
import org.qortal.data.transaction.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.transaction.Transaction.TransactionType;
import java.util.*;
public class SelfSponsorshipAlgoV2 {
private final long snapshotTimestampV1 = BlockChain.getInstance().getSelfSponsorshipAlgoV1SnapshotTimestamp();
private final long snapshotTimestampV2 = BlockChain.getInstance().getSelfSponsorshipAlgoV2SnapshotTimestamp();
private final long referenceTimestamp = BlockChain.getInstance().getReferenceTimestampBlock();
private final boolean override;
private final Repository repository;
private final String address;
private int recentAssetSendCount = 0;
private int recentSponsorshipCount = 0;
private final Set<String> assetAddresses = new LinkedHashSet<>();
private final Set<String> penaltyAddresses = new LinkedHashSet<>();
private final Set<String> sponsorAddresses = new LinkedHashSet<>();
private List<RewardShareTransactionData> sponsorshipRewardShares = new ArrayList<>();
private List<TransferAssetTransactionData> transferAssetForAddress = new ArrayList<>();
public SelfSponsorshipAlgoV2(Repository repository, String address, boolean override) {
this.repository = repository;
this.address = address;
this.override = override;
}
public String getAddress() {
return this.address;
}
public Set<String> getPenaltyAddresses() {
return this.penaltyAddresses;
}
public void run() throws DataException {
if (!override) {
this.getAccountPrivs(this.address);
}
if (override) {
this.fetchTransferAssetForAddress(this.address);
this.findRecentAssetSendCount();
if (this.recentAssetSendCount >= 6) {
this.penaltyAddresses.add(this.address);
this.penaltyAddresses.addAll(this.assetAddresses);
}
}
}
private void getAccountPrivs(String address) throws DataException {
AccountData accountData = this.repository.getAccountRepository().getAccount(address);
List<TransactionData> transferPrivsTransactions = fetchTransferPrivsForAddress(address);
transferPrivsTransactions.removeIf(t -> t.getTimestamp() > this.referenceTimestamp || accountData.getAddress().equals(t.getRecipient()));
if (transferPrivsTransactions.isEmpty()) {
// Nothing to do
return;
}
for (TransactionData transactionData : transferPrivsTransactions) {
TransferPrivsTransactionData transferPrivsTransactionData = (TransferPrivsTransactionData) transactionData;
this.penaltyAddresses.add(transferPrivsTransactionData.getRecipient());
this.fetchSponsorshipRewardShares(transferPrivsTransactionData.getRecipient());
this.findRecentSponsorshipCount();
if (this.recentSponsorshipCount >= 1) {
this.penaltyAddresses.addAll(this.sponsorAddresses);
}
String newAddress = this.getDestinationAccount(transferPrivsTransactionData.getRecipient());
while (newAddress != null) {
// Found destination account
this.penaltyAddresses.add(newAddress);
this.fetchSponsorshipRewardShares(newAddress);
this.findRecentSponsorshipCount();
if (this.recentSponsorshipCount >= 1) {
this.penaltyAddresses.addAll(this.sponsorAddresses);
}
newAddress = this.getDestinationAccount(newAddress);
}
}
}
private String getDestinationAccount(String address) throws DataException {
AccountData accountData = this.repository.getAccountRepository().getAccount(address);
List<TransactionData> transferPrivsTransactions = fetchTransferPrivsForAddress(address);
transferPrivsTransactions.removeIf(t -> t.getTimestamp() > this.referenceTimestamp || accountData.getAddress().equals(t.getRecipient()));
if (transferPrivsTransactions.isEmpty()) {
return null;
}
if (accountData == null) {
return null;
}
for (TransactionData transactionData : transferPrivsTransactions) {
TransferPrivsTransactionData transferPrivsTransactionData = (TransferPrivsTransactionData) transactionData;
if (Arrays.equals(transferPrivsTransactionData.getSenderPublicKey(), accountData.getPublicKey())) {
return transferPrivsTransactionData.getRecipient();
}
}
return null;
}
private void fetchSponsorshipRewardShares(String address) throws DataException {
AccountData accountDataRs = this.repository.getAccountRepository().getAccount(address);
List<RewardShareTransactionData> sponsorshipRewardShares = new ArrayList<>();
// Define relevant transactions
List<TransactionType> txTypes = List.of(TransactionType.REWARD_SHARE);
List<TransactionData> transactionDataList = fetchTransactions(repository, txTypes, address, false);
for (TransactionData transactionData : transactionDataList) {
if (transactionData.getType() != TransactionType.REWARD_SHARE) {
continue;
}
RewardShareTransactionData rewardShareTransactionData = (RewardShareTransactionData) transactionData;
// Skip removals
if (rewardShareTransactionData.getSharePercent() < 0) {
continue;
}
// Skip if not sponsored by this account
if (!Arrays.equals(rewardShareTransactionData.getCreatorPublicKey(), accountDataRs.getPublicKey())) {
continue;
}
// Skip self shares
if (Objects.equals(rewardShareTransactionData.getRecipient(), address)) {
continue;
}
boolean duplicateFound = false;
for (RewardShareTransactionData existingRewardShare : sponsorshipRewardShares) {
if (Objects.equals(existingRewardShare.getRecipient(), rewardShareTransactionData.getRecipient())) {
// Duplicate
duplicateFound = true;
break;
}
}
if (!duplicateFound) {
sponsorshipRewardShares.add(rewardShareTransactionData);
this.sponsorAddresses.add(rewardShareTransactionData.getRecipient());
}
}
this.sponsorshipRewardShares = sponsorshipRewardShares;
}
private void fetchTransferAssetForAddress(String address) throws DataException {
List<TransferAssetTransactionData> transferAssetForAddress = new ArrayList<>();
// Define relevant transactions
List<TransactionType> txTypes = List.of(TransactionType.TRANSFER_ASSET);
List<TransactionData> transactionDataList = fetchTransactions(repository, txTypes, address, false);
transactionDataList.removeIf(t -> t.getTimestamp() <= this.snapshotTimestampV1 || t.getTimestamp() >= this.snapshotTimestampV2);
for (TransactionData transactionData : transactionDataList) {
if (transactionData.getType() != TransactionType.TRANSFER_ASSET) {
continue;
}
TransferAssetTransactionData transferAssetTransactionData = (TransferAssetTransactionData) transactionData;
if (transferAssetTransactionData.getAssetId() == Asset.QORT) {
if (!Objects.equals(transferAssetTransactionData.getRecipient(), address)) {
// Outgoing transfer asset for this account
transferAssetForAddress.add(transferAssetTransactionData);
this.assetAddresses.add(transferAssetTransactionData.getRecipient());
}
}
}
this.transferAssetForAddress = transferAssetForAddress;
}
private void findRecentSponsorshipCount() {
int recentSponsorshipCount = 0;
for (RewardShareTransactionData rewardShare : sponsorshipRewardShares) {
if (rewardShare.getTimestamp() >= this.snapshotTimestampV1) {
recentSponsorshipCount++;
}
}
this.recentSponsorshipCount = recentSponsorshipCount;
}
private void findRecentAssetSendCount() {
int recentAssetSendCount = 0;
for (TransferAssetTransactionData assetSend : transferAssetForAddress) {
if (assetSend.getTimestamp() >= this.snapshotTimestampV1) {
recentAssetSendCount++;
}
}
this.recentAssetSendCount = recentAssetSendCount;
}
private List<TransactionData> fetchTransferPrivsForAddress(String address) throws DataException {
return fetchTransactions(repository,
List.of(TransactionType.TRANSFER_PRIVS),
address, true);
}
private static List<TransactionData> fetchTransactions(Repository repository, List<TransactionType> txTypes, String address, boolean reverse) throws DataException {
// Fetch all relevant transactions for this account
List<byte[]> signatures = repository.getTransactionRepository()
.getSignaturesMatchingCriteria(null, null, null, txTypes,
null, null, address, TransactionsResource.ConfirmationStatus.CONFIRMED,
null, null, reverse);
List<TransactionData> transactionDataList = new ArrayList<>();
for (byte[] signature : signatures) {
// Fetch transaction data
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (transactionData == null) {
continue;
}
transactionDataList.add(transactionData);
}
return transactionDataList;
}
}

View File

@ -0,0 +1,370 @@
package org.qortal.account;
import org.qortal.api.resource.TransactionsResource;
import org.qortal.asset.Asset;
import org.qortal.data.account.AccountData;
import org.qortal.data.naming.NameData;
import org.qortal.data.transaction.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.transaction.Transaction.TransactionType;
import java.util.*;
import java.util.stream.Collectors;
public class SelfSponsorshipAlgoV3 {
private final Repository repository;
private final String address;
private final AccountData accountData;
private final long snapshotTimestampV1;
private final long snapshotTimestampV3;
private final boolean override;
private int registeredNameCount = 0;
private int suspiciousCount = 0;
private int suspiciousPercent = 0;
private int consolidationCount = 0;
private int bulkIssuanceCount = 0;
private int recentSponsorshipCount = 0;
private List<RewardShareTransactionData> sponsorshipRewardShares = new ArrayList<>();
private final Map<String, List<TransactionData>> paymentsByAddress = new HashMap<>();
private final Set<String> sponsees = new LinkedHashSet<>();
private Set<String> consolidatedAddresses = new LinkedHashSet<>();
private final Set<String> zeroTransactionAddreses = new LinkedHashSet<>();
private final Set<String> penaltyAddresses = new LinkedHashSet<>();
public SelfSponsorshipAlgoV3(Repository repository, String address, long snapshotTimestampV1, long snapshotTimestampV3, boolean override) throws DataException {
this.repository = repository;
this.address = address;
this.accountData = this.repository.getAccountRepository().getAccount(this.address);
this.snapshotTimestampV1 = snapshotTimestampV1;
this.snapshotTimestampV3 = snapshotTimestampV3;
this.override = override;
}
public String getAddress() {
return this.address;
}
public Set<String> getPenaltyAddresses() {
return this.penaltyAddresses;
}
public void run() throws DataException {
if (this.accountData == null) {
// Nothing to do
return;
}
this.fetchSponsorshipRewardShares();
if (this.sponsorshipRewardShares.isEmpty()) {
// Nothing to do
return;
}
this.findConsolidatedRewards();
this.findBulkIssuance();
this.findRegisteredNameCount();
this.findRecentSponsorshipCount();
int score = this.calculateScore();
if (score <= 0 && !override) {
return;
}
String newAddress = this.getDestinationAccount(this.address);
while (newAddress != null) {
// Found destination account
this.penaltyAddresses.add(newAddress);
// Run algo for this address, but in "override" mode because it has already been flagged
SelfSponsorshipAlgoV3 algoV3 = new SelfSponsorshipAlgoV3(this.repository, newAddress, this.snapshotTimestampV1, this.snapshotTimestampV3, true);
algoV3.run();
this.penaltyAddresses.addAll(algoV3.getPenaltyAddresses());
newAddress = this.getDestinationAccount(newAddress);
}
this.penaltyAddresses.add(this.address);
if (this.override || this.recentSponsorshipCount < 20) {
this.penaltyAddresses.addAll(this.consolidatedAddresses);
this.penaltyAddresses.addAll(this.zeroTransactionAddreses);
}
else {
this.penaltyAddresses.addAll(this.sponsees);
}
}
private String getDestinationAccount(String address) throws DataException {
List<TransactionData> transferPrivsTransactions = fetchTransferPrivsForAddress(address);
if (transferPrivsTransactions.isEmpty()) {
// No TRANSFER_PRIVS transactions for this address
return null;
}
AccountData accountData = this.repository.getAccountRepository().getAccount(address);
if (accountData == null) {
return null;
}
for (TransactionData transactionData : transferPrivsTransactions) {
TransferPrivsTransactionData transferPrivsTransactionData = (TransferPrivsTransactionData) transactionData;
if (Arrays.equals(transferPrivsTransactionData.getSenderPublicKey(), accountData.getPublicKey())) {
return transferPrivsTransactionData.getRecipient();
}
}
return null;
}
private void findConsolidatedRewards() throws DataException {
List<String> sponseesThatSentRewards = new ArrayList<>();
Map<String, Integer> paymentRecipients = new HashMap<>();
// Collect outgoing payments of each sponsee
for (String sponseeAddress : this.sponsees) {
// Firstly fetch all payments for address, since the functions below depend on this data
this.fetchPaymentsForAddress(sponseeAddress);
// Check if the address has zero relevant transactions
if (this.hasZeroTransactions(sponseeAddress)) {
this.zeroTransactionAddreses.add(sponseeAddress);
}
// Get payment recipients
List<String> allPaymentRecipients = this.fetchOutgoingPaymentRecipientsForAddress(sponseeAddress);
if (allPaymentRecipients.isEmpty()) {
continue;
}
sponseesThatSentRewards.add(sponseeAddress);
List<String> addressesPaidByThisSponsee = new ArrayList<>();
for (String paymentRecipient : allPaymentRecipients) {
if (addressesPaidByThisSponsee.contains(paymentRecipient)) {
// We already tracked this association - don't allow multiple to stack up
continue;
}
addressesPaidByThisSponsee.add(paymentRecipient);
// Increment count for this recipient, or initialize to 1 if not present
if (paymentRecipients.computeIfPresent(paymentRecipient, (k, v) -> v + 1) == null) {
paymentRecipients.put(paymentRecipient, 1);
}
}
}
// Exclude addresses with a low number of payments
Map<String, Integer> filteredPaymentRecipients = paymentRecipients.entrySet().stream()
.filter(p -> p.getValue() != null && p.getValue() >= 10)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
// Now check how many sponsees have sent to this subset of addresses
Map<String, Integer> sponseesThatConsolidatedRewards = new HashMap<>();
for (String sponseeAddress : sponseesThatSentRewards) {
List<String> allPaymentRecipients = this.fetchOutgoingPaymentRecipientsForAddress(sponseeAddress);
// Remove any that aren't to one of the flagged recipients (i.e. consolidation)
allPaymentRecipients.removeIf(r -> !filteredPaymentRecipients.containsKey(r));
int count = allPaymentRecipients.size();
if (count == 0) {
continue;
}
if (sponseesThatConsolidatedRewards.computeIfPresent(sponseeAddress, (k, v) -> v + count) == null) {
sponseesThatConsolidatedRewards.put(sponseeAddress, count);
}
}
// Remove sponsees that have only sent a low number of payments to the filtered addresses
Map<String, Integer> filteredSponseesThatConsolidatedRewards = sponseesThatConsolidatedRewards.entrySet().stream()
.filter(p -> p.getValue() != null && p.getValue() >= 2)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
this.consolidationCount = sponseesThatConsolidatedRewards.size();
this.consolidatedAddresses = new LinkedHashSet<>(filteredSponseesThatConsolidatedRewards.keySet());
this.suspiciousCount = this.consolidationCount + this.zeroTransactionAddreses.size();
this.suspiciousPercent = (int)(this.suspiciousCount / (float) this.sponsees.size() * 100);
}
private void findBulkIssuance() {
Long lastTimestamp = null;
for (RewardShareTransactionData rewardShareTransactionData : sponsorshipRewardShares) {
long timestamp = rewardShareTransactionData.getTimestamp();
if (timestamp >= this.snapshotTimestampV3) {
continue;
}
if (lastTimestamp != null) {
if (timestamp - lastTimestamp < 3*60*1000L) {
this.bulkIssuanceCount++;
}
}
lastTimestamp = timestamp;
}
}
private void findRegisteredNameCount() throws DataException {
int registeredNameCount = 0;
for (String sponseeAddress : sponsees) {
List<NameData> names = repository.getNameRepository().getNamesByOwner(sponseeAddress);
for (NameData name : names) {
if (name.getRegistered() < this.snapshotTimestampV3) {
registeredNameCount++;
break;
}
}
}
this.registeredNameCount = registeredNameCount;
}
private void findRecentSponsorshipCount() {
int recentSponsorshipCount = 0;
for (RewardShareTransactionData rewardShare : sponsorshipRewardShares) {
if (rewardShare.getTimestamp() >= this.snapshotTimestampV1) {
recentSponsorshipCount++;
}
}
this.recentSponsorshipCount = recentSponsorshipCount;
}
private int calculateScore() {
final int suspiciousMultiplier = (this.suspiciousCount >= 100) ? this.suspiciousPercent : 1;
final int nameMultiplier = (this.sponsees.size() >= 25 && this.registeredNameCount <= 1) ? 21 :
(this.sponsees.size() >= 15 && this.registeredNameCount <= 1) ? 11 :
(this.sponsees.size() >= 5 && this.registeredNameCount <= 1) ? 5 : 1;
final int consolidationMultiplier = Math.max(this.consolidationCount, 1);
final int bulkIssuanceMultiplier = Math.max(this.bulkIssuanceCount / 2, 1);
final int offset = 20;
return suspiciousMultiplier * nameMultiplier * consolidationMultiplier * bulkIssuanceMultiplier - offset;
}
private void fetchSponsorshipRewardShares() throws DataException {
List<RewardShareTransactionData> sponsorshipRewardShares = new ArrayList<>();
// Define relevant transactions
List<TransactionType> txTypes = List.of(TransactionType.REWARD_SHARE);
List<TransactionData> transactionDataList = fetchTransactions(repository, txTypes, this.address, false);
transactionDataList.removeIf(t -> t.getTimestamp() <= this.snapshotTimestampV1 || t.getTimestamp() >= this.snapshotTimestampV3);
for (TransactionData transactionData : transactionDataList) {
if (transactionData.getType() != TransactionType.REWARD_SHARE) {
continue;
}
RewardShareTransactionData rewardShareTransactionData = (RewardShareTransactionData) transactionData;
// Skip removals
if (rewardShareTransactionData.getSharePercent() < 0) {
continue;
}
// Skip if not sponsored by this account
if (!Arrays.equals(rewardShareTransactionData.getCreatorPublicKey(), accountData.getPublicKey())) {
continue;
}
// Skip self shares
if (Objects.equals(rewardShareTransactionData.getRecipient(), this.address)) {
continue;
}
boolean duplicateFound = false;
for (RewardShareTransactionData existingRewardShare : sponsorshipRewardShares) {
if (Objects.equals(existingRewardShare.getRecipient(), rewardShareTransactionData.getRecipient())) {
// Duplicate
duplicateFound = true;
break;
}
}
if (!duplicateFound) {
sponsorshipRewardShares.add(rewardShareTransactionData);
this.sponsees.add(rewardShareTransactionData.getRecipient());
}
}
this.sponsorshipRewardShares = sponsorshipRewardShares;
}
private List<TransactionData> fetchTransferPrivsForAddress(String address) throws DataException {
return fetchTransactions(repository,
List.of(TransactionType.TRANSFER_PRIVS),
address, true);
}
private void fetchPaymentsForAddress(String address) throws DataException {
List<TransactionData> payments = fetchTransactions(repository,
Arrays.asList(TransactionType.PAYMENT, TransactionType.TRANSFER_ASSET),
address, false);
this.paymentsByAddress.put(address, payments);
}
private List<String> fetchOutgoingPaymentRecipientsForAddress(String address) {
List<String> outgoingPaymentRecipients = new ArrayList<>();
List<TransactionData> transactionDataList = this.paymentsByAddress.get(address);
if (transactionDataList == null) transactionDataList = new ArrayList<>();
transactionDataList.removeIf(t -> t.getTimestamp() <= this.snapshotTimestampV1 || t.getTimestamp() >= this.snapshotTimestampV3);
for (TransactionData transactionData : transactionDataList) {
switch (transactionData.getType()) {
case PAYMENT:
PaymentTransactionData paymentTransactionData = (PaymentTransactionData) transactionData;
if (!Objects.equals(paymentTransactionData.getRecipient(), address)) {
// Outgoing payment from this account
outgoingPaymentRecipients.add(paymentTransactionData.getRecipient());
}
break;
case TRANSFER_ASSET:
TransferAssetTransactionData transferAssetTransactionData = (TransferAssetTransactionData) transactionData;
if (transferAssetTransactionData.getAssetId() == Asset.QORT) {
if (!Objects.equals(transferAssetTransactionData.getRecipient(), address)) {
// Outgoing payment from this account
outgoingPaymentRecipients.add(transferAssetTransactionData.getRecipient());
}
}
break;
default:
break;
}
}
return outgoingPaymentRecipients;
}
private boolean hasZeroTransactions(String address) {
List<TransactionData> transactionDataList = this.paymentsByAddress.get(address);
if (transactionDataList == null) {
return true;
}
transactionDataList.removeIf(t -> t.getTimestamp() <= this.snapshotTimestampV1 || t.getTimestamp() >= this.snapshotTimestampV3);
return transactionDataList.isEmpty();
}
private static List<TransactionData> fetchTransactions(Repository repository, List<TransactionType> txTypes, String address, boolean reverse) throws DataException {
// Fetch all relevant transactions for this account
List<byte[]> signatures = repository.getTransactionRepository()
.getSignaturesMatchingCriteria(null, null, null, txTypes,
null, null, address, TransactionsResource.ConfirmationStatus.CONFIRMED,
null, null, reverse);
List<TransactionData> transactionDataList = new ArrayList<>();
for (byte[] signature : signatures) {
// Fetch transaction data
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (transactionData == null) {
continue;
}
transactionDataList.add(transactionData);
}
return transactionDataList;
}
}

View File

@ -1,10 +1,9 @@
package org.qortal.api;
import java.math.BigDecimal;
import org.qortal.utils.Amounts;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import org.qortal.utils.Amounts;
import java.math.BigDecimal;
public class AmountTypeAdapter extends XmlAdapter<String, Long> {

View File

@ -1,13 +1,12 @@
package org.qortal.api;
import static java.util.Arrays.stream;
import static java.util.stream.Collectors.toMap;
import java.util.Map;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.Map;
import static java.util.Arrays.stream;
import static java.util.stream.Collectors.toMap;
@XmlAccessorType(XmlAccessType.NONE)
@XmlRootElement
@ -79,7 +78,7 @@ public enum ApiError {
// BUYER_ALREADY_OWNER(411, 422),
// POLLS
// POLL_NO_EXISTS(501, 404),
POLL_NO_EXISTS(501, 404),
// POLL_ALREADY_EXISTS(502, 422),
// DUPLICATE_OPTION(503, 422),
// POLL_OPTION_NO_EXISTS(504, 404),
@ -132,7 +131,11 @@ public enum ApiError {
FOREIGN_BLOCKCHAIN_TOO_SOON(1203, 408),
// Trade portal
ORDER_SIZE_TOO_SMALL(1300, 402);
ORDER_SIZE_TOO_SMALL(1300, 402),
// Data
FILE_NOT_FOUND(1401, 404),
NO_REPLY(1402, 404);
private static final Map<Integer, ApiError> map = stream(ApiError.values()).collect(toMap(apiError -> apiError.code, apiError -> apiError));

View File

@ -1,18 +1,17 @@
package org.qortal.api;
import java.io.IOException;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.ErrorHandler;
import org.qortal.settings.Settings;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
public class ApiErrorHandler extends ErrorHandler {
private static final Logger LOGGER = LogManager.getLogger(ApiErrorHandler.class);

View File

@ -1,9 +1,9 @@
package org.qortal.api;
import javax.servlet.http.HttpServletRequest;
import org.qortal.globalization.Translator;
import javax.servlet.http.HttpServletRequest;
public enum ApiExceptionFactory {
INSTANCE;

View File

@ -0,0 +1,107 @@
package org.qortal.api;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.SecureRandom;
public class ApiKey {
private String apiKey;
public ApiKey() throws IOException {
this.load();
}
public void generate() throws IOException {
byte[] apiKey = new byte[16];
new SecureRandom().nextBytes(apiKey);
this.apiKey = Base58.encode(apiKey);
this.save();
}
/* Filesystem */
private Path getFilePath() {
return Paths.get(Settings.getInstance().getApiKeyPath(), "apikey.txt");
}
private boolean load() throws IOException {
Path path = this.getFilePath();
File apiKeyFile = new File(path.toString());
if (!apiKeyFile.exists()) {
// Try settings - to allow legacy API keys to be supported
return this.loadLegacyApiKey();
}
try {
this.apiKey = new String(Files.readAllBytes(path));
} catch (IOException e) {
throw new IOException(String.format("Couldn't read contents from file %s", path.toString()));
}
return true;
}
private boolean loadLegacyApiKey() {
String legacyApiKey = Settings.getInstance().getApiKey();
if (legacyApiKey != null && !legacyApiKey.isEmpty()) {
this.apiKey = Settings.getInstance().getApiKey();
try {
// Save it to the apikey file
this.save();
} catch (IOException e) {
// Ignore failures as it will be reloaded from settings next time
}
return true;
}
return false;
}
public void save() throws IOException {
if (this.apiKey == null || this.apiKey.isEmpty()) {
throw new IllegalStateException("Unable to save a blank API key");
}
Path filePath = this.getFilePath();
BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toString()));
writer.write(this.apiKey);
writer.close();
}
public void delete() throws IOException {
this.apiKey = null;
Path filePath = this.getFilePath();
if (Files.exists(filePath)) {
Files.delete(filePath);
}
}
public boolean generated() {
return (this.apiKey != null);
}
public boolean exists() {
return this.getFilePath().toFile().exists();
}
@Override
public String toString() {
return this.apiKey;
}
}

View File

@ -1,35 +1,24 @@
package org.qortal.api;
import org.eclipse.persistence.exceptions.XMLMarshalException;
import org.eclipse.persistence.jaxb.JAXBContextFactory;
import org.eclipse.persistence.jaxb.MarshallerProperties;
import org.eclipse.persistence.jaxb.UnmarshallerProperties;
import javax.net.ssl.*;
import javax.xml.bind.*;
import javax.xml.transform.stream.StreamSource;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.SocketAddress;
import java.net.URL;
import java.net.URLEncoder;
import java.io.Writer;
import java.net.*;
import java.util.Collections;
import java.util.Map;
import java.util.Scanner;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SNIHostName;
import javax.net.ssl.SNIServerName;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLSocket;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.UnmarshalException;
import javax.xml.bind.Unmarshaller;
import javax.xml.transform.stream.StreamSource;
import org.eclipse.persistence.exceptions.XMLMarshalException;
import org.eclipse.persistence.jaxb.JAXBContextFactory;
import org.eclipse.persistence.jaxb.UnmarshallerProperties;
public class ApiRequest {
private static final Pattern proxyUrlPattern = Pattern.compile("(https://)([^@:/]+)@([0-9.]{7,15})(/.*)");
@ -107,6 +96,36 @@ public class ApiRequest {
}
}
private static Marshaller createMarshaller(Class<?> objectClass) {
try {
// Create JAXB context aware of object's class
JAXBContext jc = JAXBContextFactory.createContext(new Class[] { objectClass }, null);
// Create marshaller
Marshaller marshaller = jc.createMarshaller();
// Set the marshaller media type to JSON
marshaller.setProperty(MarshallerProperties.MEDIA_TYPE, "application/json");
// Tell marshaller not to include JSON root element in the output
marshaller.setProperty(MarshallerProperties.JSON_INCLUDE_ROOT, false);
return marshaller;
} catch (JAXBException e) {
throw new RuntimeException("Unable to create API marshaller", e);
}
}
public static void marshall(Writer writer, Object object) throws IOException {
Marshaller marshaller = createMarshaller(object.getClass());
try {
marshaller.marshal(object, writer);
} catch (JAXBException e) {
throw new IOException("Unable to create marshall object for API", e);
}
}
public static String getParamsString(Map<String, String> params) {
StringBuilder result = new StringBuilder();
@ -122,7 +141,7 @@ public class ApiRequest {
}
String resultString = result.toString();
return resultString.length() > 0 ? resultString.substring(0, resultString.length() - 1) : resultString;
return !resultString.isEmpty() ? resultString.substring(0, resultString.length() - 1) : resultString;
}
/**

View File

@ -1,34 +1,10 @@
package org.qortal.api;
import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyStore;
import java.security.SecureRandom;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.rewrite.handler.RedirectPatternRule;
import org.eclipse.jetty.rewrite.handler.RewriteHandler;
import org.eclipse.jetty.server.CustomRequestLog;
import org.eclipse.jetty.server.DetectorConnectionFactory;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.RequestLog;
import org.eclipse.jetty.server.RequestLogWriter;
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.handler.ErrorHandler;
import org.eclipse.jetty.server.handler.InetAccessHandler;
import org.eclipse.jetty.servlet.DefaultServlet;
@ -41,27 +17,35 @@ import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.servlet.ServletContainer;
import org.qortal.api.resource.AnnotationPostProcessor;
import org.qortal.api.resource.ApiDefinition;
import org.qortal.api.websocket.ActiveChatsWebSocket;
import org.qortal.api.websocket.AdminStatusWebSocket;
import org.qortal.api.websocket.BlocksWebSocket;
import org.qortal.api.websocket.ChatMessagesWebSocket;
import org.qortal.api.websocket.PresenceWebSocket;
import org.qortal.api.websocket.TradeBotWebSocket;
import org.qortal.api.websocket.TradeOffersWebSocket;
import org.qortal.api.websocket.*;
import org.qortal.network.Network;
import org.qortal.settings.Settings;
public class ApiService {
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.servlet.http.HttpServletRequest;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyStore;
import java.security.SecureRandom;
private static final Logger LOGGER = LogManager.getLogger(ApiService.class);
public class ApiService {
private static ApiService instance;
private final ResourceConfig config;
private Server server;
private ApiKey apiKey;
public static final String API_VERSION_HEADER = "X-API-VERSION";
private ApiService() {
this.config = new ResourceConfig();
this.config.packages("org.qortal.api.resource");
this.config.packages("org.qortal.api.resource", "org.qortal.api.restricted.resource");
this.config.register(OpenApiResource.class);
this.config.register(ApiDefinition.class);
this.config.register(AnnotationPostProcessor.class);
@ -78,6 +62,15 @@ public class ApiService {
return this.config.getClasses();
}
public void setApiKey(ApiKey apiKey) {
this.apiKey = apiKey;
}
public ApiKey getApiKey() {
return this.apiKey;
}
public void start() {
try {
// Create API server
@ -92,7 +85,7 @@ public class ApiService {
throw new RuntimeException("Failed to start SSL API due to broken keystore");
// BouncyCastle-specific SSLContext build
SSLContext sslContext = SSLContext.getInstance("TLS", "BCJSSE");
SSLContext sslContext = SSLContext.getInstance("TLSv1.3", "BCJSSE");
KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC");
@ -122,13 +115,13 @@ public class ApiService {
ServerConnector portUnifiedConnector = new ServerConnector(this.server,
new DetectorConnectionFactory(sslConnectionFactory),
httpConnectionFactory);
portUnifiedConnector.setHost(Settings.getInstance().getBindAddress());
portUnifiedConnector.setHost(Network.getInstance().getBindAddress());
portUnifiedConnector.setPort(Settings.getInstance().getApiPort());
this.server.addConnector(portUnifiedConnector);
} else {
// Non-SSL
InetAddress bindAddr = InetAddress.getByName(Settings.getInstance().getBindAddress());
InetAddress bindAddr = InetAddress.getByName(Network.getInstance().getBindAddress());
InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getApiPort());
this.server = new Server(endpoint);
}
@ -205,10 +198,10 @@ public class ApiService {
context.addServlet(ChatMessagesWebSocket.class, "/websockets/chat/messages");
context.addServlet(TradeOffersWebSocket.class, "/websockets/crosschain/tradeoffers");
context.addServlet(TradeBotWebSocket.class, "/websockets/crosschain/tradebot");
context.addServlet(PresenceWebSocket.class, "/websockets/presence");
context.addServlet(TradePresenceWebSocket.class, "/websockets/crosschain/tradepresence");
// Warn about API security if needed
this.checkApiSecurity();
// Deprecated
context.addServlet(PresenceWebSocket.class, "/websockets/presence");
// Start server
this.server.start();
@ -229,23 +222,19 @@ public class ApiService {
this.server = null;
}
private void checkApiSecurity() {
// Warn about API security if needed
boolean allConnectionsAllowed = false;
if (Settings.getInstance().isApiKeyDisabled()) {
for (String pattern : Settings.getInstance().getApiWhitelist()) {
if (pattern.startsWith("0.0.0.0/") || pattern.startsWith("::/") || pattern.endsWith("/0")) {
allConnectionsAllowed = true;
}
}
if (allConnectionsAllowed) {
LOGGER.warn("Warning: API key validation is currently disabled, and the API whitelist " +
"is allowing all connections. This can be a security risk.");
LOGGER.warn("To fix, set the apiKeyDisabled setting to false, or allow only specific local " +
"IP addresses using the apiWhitelist setting.");
}
public static int getApiVersion(HttpServletRequest request) {
// Get API version
String apiVersionString = request.getHeader(API_VERSION_HEADER);
if (apiVersionString == null) {
// Try query string - this is needed to avoid a CORS preflight. See: https://stackoverflow.com/a/43881141
apiVersionString = request.getParameter("apiVersion");
}
int apiVersion = 1;
if (apiVersionString != null) {
apiVersion = Integer.parseInt(apiVersionString);
}
return apiVersion;
}
}

View File

@ -1,9 +1,9 @@
package org.qortal.api;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import org.qortal.utils.Base58;
import javax.xml.bind.annotation.adapters.XmlAdapter;
public class Base58TypeAdapter extends XmlAdapter<String, byte[]> {
@Override

View File

@ -1,8 +1,7 @@
package org.qortal.api;
import java.math.BigDecimal;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.math.BigDecimal;
public class BigDecimalTypeAdapter extends XmlAdapter<String, BigDecimal> {

View File

@ -4,9 +4,11 @@ import io.swagger.v3.oas.models.Operation;
import io.swagger.v3.oas.models.PathItem;
import io.swagger.v3.oas.models.info.Info;
import io.swagger.v3.oas.models.responses.ApiResponse;
import static java.util.Arrays.asList;
import java.util.List;
import static java.util.Arrays.asList;
class Constants {
public static final String APIERROR_CONTEXT_PATH = "/Api";
public static final String APIERROR_KEY = "ApiError/%s";

View File

@ -0,0 +1,173 @@
package org.qortal.api;
import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.rewrite.handler.RewriteHandler;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.handler.ErrorHandler;
import org.eclipse.jetty.server.handler.InetAccessHandler;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.servlets.CrossOriginFilter;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.servlet.ServletContainer;
import org.qortal.api.resource.AnnotationPostProcessor;
import org.qortal.api.resource.ApiDefinition;
import org.qortal.network.Network;
import org.qortal.repository.DataException;
import org.qortal.settings.Settings;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyStore;
import java.security.SecureRandom;
public class DevProxyService {
private static DevProxyService instance;
private final ResourceConfig config;
private Server server;
private DevProxyService() {
this.config = new ResourceConfig();
this.config.packages("org.qortal.api.proxy.resource", "org.qortal.api.resource");
this.config.register(OpenApiResource.class);
this.config.register(ApiDefinition.class);
this.config.register(AnnotationPostProcessor.class);
}
public static DevProxyService getInstance() {
if (instance == null)
instance = new DevProxyService();
return instance;
}
public Iterable<Class<?>> getResources() {
return this.config.getClasses();
}
public void start() throws DataException {
try {
// Create API server
// SSL support if requested
String keystorePathname = Settings.getInstance().getSslKeystorePathname();
String keystorePassword = Settings.getInstance().getSslKeystorePassword();
if (keystorePathname != null && keystorePassword != null) {
// SSL version
if (!Files.isReadable(Path.of(keystorePathname)))
throw new RuntimeException("Failed to start SSL API due to broken keystore");
// BouncyCastle-specific SSLContext build
SSLContext sslContext = SSLContext.getInstance("TLS", "BCJSSE");
KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC");
try (InputStream keystoreStream = Files.newInputStream(Paths.get(keystorePathname))) {
keyStore.load(keystoreStream, keystorePassword.toCharArray());
}
keyManagerFactory.init(keyStore, keystorePassword.toCharArray());
sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom());
SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
sslContextFactory.setSslContext(sslContext);
this.server = new Server();
HttpConfiguration httpConfig = new HttpConfiguration();
httpConfig.setSecureScheme("https");
httpConfig.setSecurePort(Settings.getInstance().getDevProxyPort());
SecureRequestCustomizer src = new SecureRequestCustomizer();
httpConfig.addCustomizer(src);
HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig);
SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString());
ServerConnector portUnifiedConnector = new ServerConnector(this.server,
new DetectorConnectionFactory(sslConnectionFactory),
httpConnectionFactory);
portUnifiedConnector.setHost(Network.getInstance().getBindAddress());
portUnifiedConnector.setPort(Settings.getInstance().getDevProxyPort());
this.server.addConnector(portUnifiedConnector);
} else {
// Non-SSL
InetAddress bindAddr = InetAddress.getByName(Network.getInstance().getBindAddress());
InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getDevProxyPort());
this.server = new Server(endpoint);
}
// Error handler
ErrorHandler errorHandler = new ApiErrorHandler();
this.server.setErrorHandler(errorHandler);
// Request logging
if (Settings.getInstance().isDevProxyLoggingEnabled()) {
RequestLogWriter logWriter = new RequestLogWriter("devproxy-requests.log");
logWriter.setAppend(true);
logWriter.setTimeZone("UTC");
RequestLog requestLog = new CustomRequestLog(logWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT);
this.server.setRequestLog(requestLog);
}
// Access handler (currently no whitelist is used)
InetAccessHandler accessHandler = new InetAccessHandler();
this.server.setHandler(accessHandler);
// URL rewriting
RewriteHandler rewriteHandler = new RewriteHandler();
accessHandler.setHandler(rewriteHandler);
// Context
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
context.setContextPath("/");
rewriteHandler.setHandler(context);
// Cross-origin resource sharing
FilterHolder corsFilterHolder = new FilterHolder(CrossOriginFilter.class);
corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*");
corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET, POST, DELETE");
corsFilterHolder.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false");
context.addFilter(corsFilterHolder, "/*", null);
// API servlet
ServletContainer container = new ServletContainer(this.config);
ServletHolder apiServlet = new ServletHolder(container);
apiServlet.setInitOrder(1);
context.addServlet(apiServlet, "/*");
// Start server
this.server.start();
} catch (Exception e) {
// Failed to start
throw new DataException("Failed to start developer proxy", e);
}
}
public void stop() {
try {
// Stop server
this.server.stop();
} catch (Exception e) {
// Failed to stop
}
this.server = null;
instance = null;
}
}

View File

@ -0,0 +1,171 @@
package org.qortal.api;
import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.rewrite.handler.RewriteHandler;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.handler.ErrorHandler;
import org.eclipse.jetty.server.handler.InetAccessHandler;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.servlets.CrossOriginFilter;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.servlet.ServletContainer;
import org.qortal.api.resource.AnnotationPostProcessor;
import org.qortal.api.resource.ApiDefinition;
import org.qortal.network.Network;
import org.qortal.settings.Settings;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyStore;
import java.security.SecureRandom;
public class DomainMapService {
private static DomainMapService instance;
private final ResourceConfig config;
private Server server;
private DomainMapService() {
this.config = new ResourceConfig();
this.config.packages("org.qortal.api.resource", "org.qortal.api.domainmap.resource");
this.config.register(OpenApiResource.class);
this.config.register(ApiDefinition.class);
this.config.register(AnnotationPostProcessor.class);
}
public static DomainMapService getInstance() {
if (instance == null)
instance = new DomainMapService();
return instance;
}
public Iterable<Class<?>> getResources() {
return this.config.getClasses();
}
public void start() {
try {
// Create API server
// SSL support if requested
String keystorePathname = Settings.getInstance().getSslKeystorePathname();
String keystorePassword = Settings.getInstance().getSslKeystorePassword();
if (keystorePathname != null && keystorePassword != null) {
// SSL version
if (!Files.isReadable(Path.of(keystorePathname)))
throw new RuntimeException("Failed to start SSL API due to broken keystore");
// BouncyCastle-specific SSLContext build
SSLContext sslContext = SSLContext.getInstance("TLSv1.3", "BCJSSE");
KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC");
try (InputStream keystoreStream = Files.newInputStream(Paths.get(keystorePathname))) {
keyStore.load(keystoreStream, keystorePassword.toCharArray());
}
keyManagerFactory.init(keyStore, keystorePassword.toCharArray());
sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom());
SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
sslContextFactory.setSslContext(sslContext);
this.server = new Server();
HttpConfiguration httpConfig = new HttpConfiguration();
httpConfig.setSecureScheme("https");
httpConfig.setSecurePort(Settings.getInstance().getDomainMapPort());
SecureRequestCustomizer src = new SecureRequestCustomizer();
httpConfig.addCustomizer(src);
HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig);
SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString());
ServerConnector portUnifiedConnector = new ServerConnector(this.server,
new DetectorConnectionFactory(sslConnectionFactory),
httpConnectionFactory);
portUnifiedConnector.setHost(Network.getInstance().getBindAddress());
portUnifiedConnector.setPort(Settings.getInstance().getDomainMapPort());
this.server.addConnector(portUnifiedConnector);
} else {
// Non-SSL
InetAddress bindAddr = InetAddress.getByName(Network.getInstance().getBindAddress());
InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getDomainMapPort());
this.server = new Server(endpoint);
}
// Error handler
ErrorHandler errorHandler = new ApiErrorHandler();
this.server.setErrorHandler(errorHandler);
// Request logging
if (Settings.getInstance().isDomainMapLoggingEnabled()) {
RequestLogWriter logWriter = new RequestLogWriter("domainmap-requests.log");
logWriter.setAppend(true);
logWriter.setTimeZone("UTC");
RequestLog requestLog = new CustomRequestLog(logWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT);
this.server.setRequestLog(requestLog);
}
// Access handler (currently no whitelist is used)
InetAccessHandler accessHandler = new InetAccessHandler();
this.server.setHandler(accessHandler);
// URL rewriting
RewriteHandler rewriteHandler = new RewriteHandler();
accessHandler.setHandler(rewriteHandler);
// Context
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
context.setContextPath("/");
rewriteHandler.setHandler(context);
// Cross-origin resource sharing
FilterHolder corsFilterHolder = new FilterHolder(CrossOriginFilter.class);
corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*");
corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET, POST, DELETE");
corsFilterHolder.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false");
context.addFilter(corsFilterHolder, "/*", null);
// API servlet
ServletContainer container = new ServletContainer(this.config);
ServletHolder apiServlet = new ServletHolder(container);
apiServlet.setInitOrder(1);
context.addServlet(apiServlet, "/*");
// Start server
this.server.start();
} catch (Exception e) {
// Failed to start
throw new RuntimeException("Failed to start API", e);
}
}
public void stop() {
try {
// Stop server
this.server.stop();
} catch (Exception e) {
// Failed to stop
}
this.server = null;
}
}

View File

@ -0,0 +1,171 @@
package org.qortal.api;
import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.rewrite.handler.RewriteHandler;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.handler.ErrorHandler;
import org.eclipse.jetty.server.handler.InetAccessHandler;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.servlets.CrossOriginFilter;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.servlet.ServletContainer;
import org.qortal.api.resource.AnnotationPostProcessor;
import org.qortal.api.resource.ApiDefinition;
import org.qortal.network.Network;
import org.qortal.settings.Settings;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyStore;
import java.security.SecureRandom;
public class GatewayService {
private static GatewayService instance;
private final ResourceConfig config;
private Server server;
private GatewayService() {
this.config = new ResourceConfig();
this.config.packages("org.qortal.api.resource", "org.qortal.api.gateway.resource");
this.config.register(OpenApiResource.class);
this.config.register(ApiDefinition.class);
this.config.register(AnnotationPostProcessor.class);
}
public static GatewayService getInstance() {
if (instance == null)
instance = new GatewayService();
return instance;
}
public Iterable<Class<?>> getResources() {
return this.config.getClasses();
}
public void start() {
try {
// Create API server
// SSL support if requested
String keystorePathname = Settings.getInstance().getSslKeystorePathname();
String keystorePassword = Settings.getInstance().getSslKeystorePassword();
if (keystorePathname != null && keystorePassword != null) {
// SSL version
if (!Files.isReadable(Path.of(keystorePathname)))
throw new RuntimeException("Failed to start SSL API due to broken keystore");
// BouncyCastle-specific SSLContext build
SSLContext sslContext = SSLContext.getInstance("TLSv1.3", "BCJSSE");
KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC");
try (InputStream keystoreStream = Files.newInputStream(Paths.get(keystorePathname))) {
keyStore.load(keystoreStream, keystorePassword.toCharArray());
}
keyManagerFactory.init(keyStore, keystorePassword.toCharArray());
sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom());
SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
sslContextFactory.setSslContext(sslContext);
this.server = new Server();
HttpConfiguration httpConfig = new HttpConfiguration();
httpConfig.setSecureScheme("https");
httpConfig.setSecurePort(Settings.getInstance().getGatewayPort());
SecureRequestCustomizer src = new SecureRequestCustomizer();
httpConfig.addCustomizer(src);
HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig);
SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString());
ServerConnector portUnifiedConnector = new ServerConnector(this.server,
new DetectorConnectionFactory(sslConnectionFactory),
httpConnectionFactory);
portUnifiedConnector.setHost(Network.getInstance().getBindAddress());
portUnifiedConnector.setPort(Settings.getInstance().getGatewayPort());
this.server.addConnector(portUnifiedConnector);
} else {
// Non-SSL
InetAddress bindAddr = InetAddress.getByName(Network.getInstance().getBindAddress());
InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getGatewayPort());
this.server = new Server(endpoint);
}
// Error handler
ErrorHandler errorHandler = new ApiErrorHandler();
this.server.setErrorHandler(errorHandler);
// Request logging
if (Settings.getInstance().isGatewayLoggingEnabled()) {
RequestLogWriter logWriter = new RequestLogWriter("gateway-requests.log");
logWriter.setAppend(true);
logWriter.setTimeZone("UTC");
RequestLog requestLog = new CustomRequestLog(logWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT);
this.server.setRequestLog(requestLog);
}
// Access handler (currently no whitelist is used)
InetAccessHandler accessHandler = new InetAccessHandler();
this.server.setHandler(accessHandler);
// URL rewriting
RewriteHandler rewriteHandler = new RewriteHandler();
accessHandler.setHandler(rewriteHandler);
// Context
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
context.setContextPath("/");
rewriteHandler.setHandler(context);
// Cross-origin resource sharing
FilterHolder corsFilterHolder = new FilterHolder(CrossOriginFilter.class);
corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*");
corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET, POST, DELETE");
corsFilterHolder.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false");
context.addFilter(corsFilterHolder, "/*", null);
// API servlet
ServletContainer container = new ServletContainer(this.config);
ServletHolder apiServlet = new ServletHolder(container);
apiServlet.setInitOrder(1);
context.addServlet(apiServlet, "/*");
// Start server
this.server.start();
} catch (Exception e) {
// Failed to start
throw new RuntimeException("Failed to start API", e);
}
}
public void stop() {
try {
// Stop server
this.server.stop();
} catch (Exception e) {
// Failed to stop
}
this.server = null;
}
}

View File

@ -0,0 +1,94 @@
package org.qortal.api;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.qortal.arbitrary.misc.Service;
import java.util.Objects;
public class HTMLParser {
private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class);
private String qdnBase;
private String qdnBaseWithPath;
private byte[] data;
private String qdnContext;
private String resourceId;
private Service service;
private String identifier;
private String path;
private String theme;
private boolean usingCustomRouting;
public HTMLParser(String resourceId, String inPath, String prefix, boolean includeResourceIdInPrefix, byte[] data,
String qdnContext, Service service, String identifier, String theme, boolean usingCustomRouting) {
String inPathWithoutFilename = inPath.contains("/") ? inPath.substring(0, inPath.lastIndexOf('/')) : String.format("/%s",inPath);
this.qdnBase = includeResourceIdInPrefix ? String.format("%s/%s", prefix, resourceId) : prefix;
this.qdnBaseWithPath = includeResourceIdInPrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : String.format("%s%s", prefix, inPathWithoutFilename);
this.data = data;
this.qdnContext = qdnContext;
this.resourceId = resourceId;
this.service = service;
this.identifier = identifier;
this.path = inPath;
this.theme = theme;
this.usingCustomRouting = usingCustomRouting;
}
public void addAdditionalHeaderTags() {
String fileContents = new String(data);
Document document = Jsoup.parse(fileContents);
Elements head = document.getElementsByTag("head");
if (!head.isEmpty()) {
// Add q-apps script tag
String qAppsScriptElement = String.format("<script src=\"/apps/q-apps.js?time=%d\">", System.currentTimeMillis());
head.get(0).prepend(qAppsScriptElement);
// Add q-apps gateway script tag if in gateway mode
if (Objects.equals(this.qdnContext, "gateway")) {
String qAppsGatewayScriptElement = String.format("<script src=\"/apps/q-apps-gateway.js?time=%d\">", System.currentTimeMillis());
head.get(0).prepend(qAppsGatewayScriptElement);
}
// Escape and add vars
String qdnContext = this.qdnContext != null ? this.qdnContext.replace("\\", "").replace("\"","\\\"") : "";
String service = this.service.toString().replace("\\", "").replace("\"","\\\"");
String name = this.resourceId != null ? this.resourceId.replace("\\", "").replace("\"","\\\"") : "";
String identifier = this.identifier != null ? this.identifier.replace("\\", "").replace("\"","\\\"") : "";
String path = this.path != null ? this.path.replace("\\", "").replace("\"","\\\"") : "";
String theme = this.theme != null ? this.theme.replace("\\", "").replace("\"","\\\"") : "";
String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\\", "").replace("\"","\\\"") : "";
String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\\", "").replace("\"","\\\"") : "";
String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\"; var _qdnBaseWithPath=\"%s\";</script>", qdnContext, theme, service, name, identifier, path, qdnBase, qdnBaseWithPath);
head.get(0).prepend(qdnContextVar);
// Add base href tag
// Exclude the path if this request was routed back to the index automatically
String baseHref = this.usingCustomRouting ? this.qdnBase : this.qdnBaseWithPath;
String baseElement = String.format("<base href=\"%s/\">", baseHref);
head.get(0).prepend(baseElement);
// Add meta charset tag
String metaCharsetElement = "<meta charset=\"UTF-8\">";
head.get(0).prepend(metaCharsetElement);
}
String html = document.html();
this.data = html.getBytes();
}
public static boolean isHtmlFile(String path) {
if (path.endsWith(".html") || path.endsWith(".htm") || path.isEmpty()) {
return true;
}
return false;
}
public byte[] getData() {
return this.data;
}
}

View File

@ -1,8 +1,7 @@
package org.qortal.api;
import java.math.BigDecimal;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.math.BigDecimal;
public class RewardSharePercentTypeAdapter extends XmlAdapter<String, Integer> {

View File

@ -0,0 +1,6 @@
package org.qortal.api;
public enum SearchMode {
LATEST,
ALL
}

View File

@ -1,38 +1,127 @@
package org.qortal.api;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.qortal.arbitrary.ArbitraryDataResource;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataRenderManager;
import org.qortal.settings.Settings;
import javax.servlet.http.HttpServletRequest;
import org.qortal.settings.Settings;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
public abstract class Security {
public static final String API_KEY_HEADER = "X-API-KEY";
/**
* Check API call is allowed, retrieving the API key from the request header or GET/POST parameters where required
* @param request
*/
public static void checkApiCallAllowed(HttpServletRequest request) {
// If API key checking has been disabled, we will allow the request in all cases
boolean isApiKeyDisabled = Settings.getInstance().isApiKeyDisabled();
if (isApiKeyDisabled)
return;
checkApiCallAllowed(request, null);
}
String expectedApiKey = Settings.getInstance().getApiKey();
String passedApiKey = request.getHeader(API_KEY_HEADER);
/**
* Check API call is allowed, retrieving the API key first from the passedApiKey parameter, with a fallback
* to the request header or GET/POST parameters when null.
* @param request
* @param passedApiKey - the API key to test, or null if it should be retrieved from the request headers.
*/
public static void checkApiCallAllowed(HttpServletRequest request, String passedApiKey) {
// We may want to allow automatic authentication for local requests, if enabled in settings
boolean localAuthBypassEnabled = Settings.getInstance().isLocalAuthBypassEnabled();
if (localAuthBypassEnabled) {
try {
InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr());
if (remoteAddr.isLoopbackAddress()) {
// Request originates from loopback address, so allow it
return;
}
} catch (UnknownHostException e) {
// Ignore failure, and fallback to API key authentication
}
}
if ((expectedApiKey != null && !expectedApiKey.equals(passedApiKey)) ||
(passedApiKey != null && !passedApiKey.equals(expectedApiKey)))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
// Retrieve the API key
ApiKey apiKey = Security.getApiKey(request);
if (!apiKey.generated()) {
// Not generated an API key yet, so disallow sensitive API calls
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "API key not generated");
}
InetAddress remoteAddr;
// We require an API key to be passed
if (passedApiKey == null) {
// API call not passed as a parameter, so try the header
passedApiKey = request.getHeader(API_KEY_HEADER);
}
if (passedApiKey == null) {
// Try query string - this is needed to avoid a CORS preflight. See: https://stackoverflow.com/a/43881141
passedApiKey = request.getParameter("apiKey");
}
if (passedApiKey == null) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Missing 'X-API-KEY' header");
}
// The API keys must match
if (!apiKey.toString().equals(passedApiKey)) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "API key invalid");
}
}
public static void disallowLoopbackRequests(HttpServletRequest request) {
try {
remoteAddr = InetAddress.getByName(request.getRemoteAddr());
InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr());
if (remoteAddr.isLoopbackAddress() && !Settings.getInstance().isGatewayLoopbackEnabled()) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Local requests not allowed");
}
} catch (UnknownHostException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
}
}
if (!remoteAddr.isLoopbackAddress())
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
public static void disallowLoopbackRequestsIfAuthBypassEnabled(HttpServletRequest request) {
if (Settings.getInstance().isLocalAuthBypassEnabled()) {
try {
InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr());
if (remoteAddr.isLoopbackAddress()) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Local requests not allowed when localAuthBypassEnabled is enabled in settings");
}
} catch (UnknownHostException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
}
}
}
public static void requirePriorAuthorization(HttpServletRequest request, String resourceId, Service service, String identifier) {
ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, identifier);
if (!ArbitraryDataRenderManager.getInstance().isAuthorized(resource)) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Call /render/authorize first");
}
}
public static void requirePriorAuthorizationOrApiKey(HttpServletRequest request, String resourceId, Service service, String identifier, String apiKey) {
try {
Security.checkApiCallAllowed(request, apiKey);
} catch (ApiException e) {
// API call wasn't allowed, but maybe it was pre-authorized
Security.requirePriorAuthorization(request, resourceId, service, identifier);
}
}
public static ApiKey getApiKey(HttpServletRequest request) {
ApiKey apiKey = ApiService.getInstance().getApiKey();
if (apiKey == null) {
try {
apiKey = new ApiKey();
} catch (IOException e) {
// Couldn't load API key - so we need to treat it as not generated, and therefore unauthorized
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
}
ApiService.getInstance().setApiKey(apiKey);
}
return apiKey;
}
}

View File

@ -1,18 +1,17 @@
package org.qortal.api;
import org.eclipse.persistence.oxm.annotations.XmlVariableNode;
import org.qortal.transaction.Transaction.TransactionType;
import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlValue;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.util.ArrayList;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlValue;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import org.eclipse.persistence.oxm.annotations.XmlVariableNode;
import org.qortal.transaction.Transaction.TransactionType;
public class TransactionCountMapXmlAdapter extends XmlAdapter<TransactionCountMapXmlAdapter.StringIntegerMap, Map<TransactionType, Integer>> {
public static class StringIntegerMap {

View File

@ -0,0 +1,58 @@
package org.qortal.api.domainmap.resource;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
import org.qortal.arbitrary.ArbitraryDataRenderer;
import org.qortal.arbitrary.misc.Service;
import org.qortal.settings.Settings;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.core.Context;
import java.util.Map;
@Path("/")
@Tag(name = "Domain Map")
public class DomainMapResource {
@Context HttpServletRequest request;
@Context HttpServletResponse response;
@Context ServletContext context;
@GET
public HttpServletResponse getIndexByDomainMap() {
return this.getDomainMap("/");
}
@GET
@Path("{path:.*}")
public HttpServletResponse getPathByDomainMap(@PathParam("path") String inPath) {
return this.getDomainMap(inPath);
}
private HttpServletResponse getDomainMap(String inPath) {
Map<String, String> domainMap = Settings.getInstance().getSimpleDomainMap();
if (domainMap != null && domainMap.containsKey(request.getServerName())) {
// Build synchronously, so that we don't need to make the summary API endpoints available over
// the domain map server. This means that there will be no loading screen, but this is potentially
// preferred in this situation anyway (e.g. to avoid confusing search engine robots).
return this.get(domainMap.get(request.getServerName()), ResourceIdType.NAME, Service.WEBSITE, null, inPath, null, "", false, false);
}
return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found");
}
private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String identifier,
String inPath, String secret58, String prefix, boolean includeResourceIdInPrefix, boolean async) {
ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, identifier, inPath,
secret58, prefix, includeResourceIdInPrefix, async, "domainMap", request, response, context);
return renderer.render();
}
}

View File

@ -0,0 +1,152 @@
package org.qortal.api.gateway.resource;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.commons.lang3.StringUtils;
import org.qortal.api.ApiError;
import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.Security;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.ArbitraryDataRenderer;
import org.qortal.arbitrary.ArbitraryDataResource;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.core.Context;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
@Path("/")
@Tag(name = "Gateway")
public class GatewayResource {
@Context HttpServletRequest request;
@Context HttpServletResponse response;
@Context ServletContext context;
private ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build) {
// If "build=true" has been specified in the query string, build the resource before returning its status
if (build != null && build) {
try {
ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null);
if (!reader.isBuilding()) {
reader.loadSynchronously(false);
}
} catch (Exception e) {
// No need to handle exception, as it will be reflected in the status
}
}
try (final Repository repository = RepositoryManager.getRepository()) {
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
return resource.getStatus(repository);
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
@GET
public HttpServletResponse getRoot() {
return ArbitraryDataRenderer.getResponse(response, 200, "");
}
@GET
@Path("{path:.*}")
@SecurityRequirement(name = "apiKey")
public HttpServletResponse getPath(@PathParam("path") String inPath) {
// Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data
Security.disallowLoopbackRequests(request);
return this.parsePath(inPath, "gateway", null, true, true);
}
private HttpServletResponse parsePath(String inPath, String qdnContext, String secret58, boolean includeResourceIdInPrefix, boolean async) {
if (inPath == null || inPath.isEmpty()) {
// Assume not a real file
return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found");
}
// Default service is WEBSITE
Service service = Service.WEBSITE;
String name = null;
String identifier = null;
String outPath = "";
List<String> prefixParts = new ArrayList<>();
if (!inPath.contains("/")) {
// Assume entire inPath is a registered name
name = inPath;
}
else {
// Parse the path to determine what we need to load
List<String> parts = new LinkedList<>(Arrays.asList(inPath.split("/")));
// Check if the first element is a service
try {
Service parsedService = Service.valueOf(parts.get(0).toUpperCase());
if (parsedService != null) {
// First element matches a service, so we can assume it is one
service = parsedService;
parts.remove(0);
prefixParts.add(service.name());
}
} catch (IllegalArgumentException e) {
// Not a service
}
if (parts.isEmpty()) {
// We need more than just a service
return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found");
}
// Service is removed, so assume first element is now a registered name
name = parts.get(0);
parts.remove(0);
if (!parts.isEmpty()) {
// Name is removed, so check if the first element is now an identifier
ArbitraryResourceStatus status = this.getStatus(service, name, parts.get(0), false);
if (status.getTotalChunkCount() > 0) {
// Matched service, name and identifier combination - so assume this is an identifier and can be removed
identifier = parts.get(0);
parts.remove(0);
prefixParts.add(identifier);
}
}
if (!parts.isEmpty()) {
// outPath can be built by combining any remaining parts
outPath = String.join("/", parts);
}
}
String prefix = StringUtils.join(prefixParts, "/");
if (prefix != null && !prefix.isEmpty()) {
prefix = "/" + prefix;
}
ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(name, ResourceIdType.NAME, service, identifier, outPath,
secret58, prefix, includeResourceIdInPrefix, async, qdnContext, request, response, context);
return renderer.render();
}
}

View File

@ -0,0 +1,54 @@
package org.qortal.api.model;
import org.qortal.block.SelfSponsorshipAlgoV1Block;
import org.qortal.data.account.AccountData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.ArrayList;
import java.util.List;
@XmlAccessorType(XmlAccessType.FIELD)
public class AccountPenaltyStats {
public Integer totalPenalties;
public Integer maxPenalty;
public Integer minPenalty;
public String penaltyHash;
protected AccountPenaltyStats() {
}
public AccountPenaltyStats(Integer totalPenalties, Integer maxPenalty, Integer minPenalty, String penaltyHash) {
this.totalPenalties = totalPenalties;
this.maxPenalty = maxPenalty;
this.minPenalty = minPenalty;
this.penaltyHash = penaltyHash;
}
public static AccountPenaltyStats fromAccounts(List<AccountData> accounts) {
int totalPenalties = 0;
Integer maxPenalty = null;
Integer minPenalty = null;
List<String> addresses = new ArrayList<>();
for (AccountData accountData : accounts) {
int penalty = accountData.getBlocksMintedPenalty();
addresses.add(accountData.getAddress());
totalPenalties++;
// Penalties are expressed as a negative number, so the min and the max are reversed here
if (maxPenalty == null || penalty < maxPenalty) maxPenalty = penalty;
if (minPenalty == null || penalty > minPenalty) minPenalty = penalty;
}
String penaltyHash = SelfSponsorshipAlgoV1Block.getHash(addresses);
return new AccountPenaltyStats(totalPenalties, maxPenalty, minPenalty, penaltyHash);
}
@Override
public String toString() {
return String.format("totalPenalties: %d, maxPenalty: %d, minPenalty: %d, penaltyHash: %s", totalPenalties, maxPenalty, minPenalty, penaltyHash == null ? "null" : penaltyHash);
}
}

View File

@ -1,15 +1,14 @@
package org.qortal.api.model;
import java.util.Collections;
import java.util.EnumMap;
import java.util.Map;
import org.qortal.api.TransactionCountMapXmlAdapter;
import org.qortal.transaction.Transaction.TransactionType;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.qortal.api.TransactionCountMapXmlAdapter;
import org.qortal.transaction.Transaction.TransactionType;
import java.util.Collections;
import java.util.EnumMap;
import java.util.Map;
@XmlAccessorType(XmlAccessType.FIELD)
public class ActivitySummary {

View File

@ -1,13 +1,13 @@
package org.qortal.api.model;
import org.qortal.data.asset.OrderData;
import javax.xml.bind.Marshaller;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.qortal.data.asset.OrderData;
@XmlAccessorType(XmlAccessType.NONE)
public class AggregatedOrder {

View File

@ -0,0 +1,101 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import org.bouncycastle.util.encoders.Base64;
import org.bouncycastle.util.encoders.DecoderException;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlTransient;
@XmlAccessorType(XmlAccessType.FIELD)
public class AtCreationRequest {
@Schema(description = "CIYAM AT version", example = "2")
private short ciyamAtVersion;
@Schema(description = "base64-encoded code bytes", example = "")
private String codeBytesBase64;
@Schema(description = "base64-encoded data bytes", example = "")
private String dataBytesBase64;
private short numCallStackPages;
private short numUserStackPages;
private long minActivationAmount;
// Default constructor for JSON deserialization
public AtCreationRequest() {}
// Getters and setters
public short getCiyamAtVersion() {
return ciyamAtVersion;
}
public void setCiyamAtVersion(short ciyamAtVersion) {
this.ciyamAtVersion = ciyamAtVersion;
}
public String getCodeBytesBase64() {
return this.codeBytesBase64;
}
@XmlTransient
@Schema(hidden = true)
public byte[] getCodeBytes() {
if (this.codeBytesBase64 != null) {
try {
return Base64.decode(this.codeBytesBase64);
}
catch (DecoderException e) {
return null;
}
}
return null;
}
public String getDataBytesBase64() {
return this.dataBytesBase64;
}
@XmlTransient
@Schema(hidden = true)
public byte[] getDataBytes() {
if (this.dataBytesBase64 != null) {
try {
return Base64.decode(this.dataBytesBase64);
}
catch (DecoderException e) {
return null;
}
}
return null;
}
public short getNumCallStackPages() {
return numCallStackPages;
}
public void setNumCallStackPages(short numCallStackPages) {
this.numCallStackPages = numCallStackPages;
}
public short getNumUserStackPages() {
return numUserStackPages;
}
public void setNumUserStackPages(short numUserStackPages) {
this.numUserStackPages = numUserStackPages;
}
public long getMinActivationAmount() {
return minActivationAmount;
}
public void setMinActivationAmount(long minActivationAmount) {
this.minActivationAmount = minActivationAmount;
}
}

View File

@ -1,10 +1,10 @@
package org.qortal.api.model;
import org.qortal.crypto.Crypto;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import org.qortal.crypto.Crypto;
@XmlAccessorType(XmlAccessType.FIELD)
public class BlockSignerSummary {

View File

@ -1,7 +1,8 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.data.network.PeerChainTipData;
import org.qortal.controller.Controller;
import org.qortal.data.block.BlockSummaryData;
import org.qortal.data.network.PeerData;
import org.qortal.network.Handshake;
import org.qortal.network.Peer;
@ -16,7 +17,7 @@ public class ConnectedPeer {
public enum Direction {
INBOUND,
OUTBOUND;
OUTBOUND
}
public Direction direction;
@ -36,6 +37,7 @@ public class ConnectedPeer {
public Long lastBlockTimestamp;
public UUID connectionId;
public String age;
public Boolean isTooDivergent;
protected ConnectedPeer() {
}
@ -63,11 +65,16 @@ public class ConnectedPeer {
this.age = "connecting...";
}
PeerChainTipData peerChainTipData = peer.getChainTipData();
BlockSummaryData peerChainTipData = peer.getChainTipData();
if (peerChainTipData != null) {
this.lastHeight = peerChainTipData.getLastHeight();
this.lastBlockSignature = peerChainTipData.getLastBlockSignature();
this.lastBlockTimestamp = peerChainTipData.getLastBlockTimestamp();
this.lastHeight = peerChainTipData.getHeight();
this.lastBlockSignature = peerChainTipData.getSignature();
this.lastBlockTimestamp = peerChainTipData.getTimestamp();
}
// Only include isTooDivergent decision if we've had the opportunity to request block summaries this peer
if (peer.getLastTooDivergentTime() != null) {
this.isTooDivergent = Controller.wasRecentlyTooDivergent.test(peer);
}
}

View File

@ -1,11 +1,10 @@
package org.qortal.api.model;
import java.math.BigDecimal;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
import java.math.BigDecimal;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainBitcoinRedeemRequest {

View File

@ -1,11 +1,10 @@
package org.qortal.api.model;
import java.math.BigDecimal;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
import java.math.BigDecimal;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainBitcoinRefundRequest {

View File

@ -1,10 +1,10 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainBitcoinTemplateRequest {

View File

@ -1,11 +1,10 @@
package org.qortal.api.model;
import java.math.BigDecimal;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
import java.math.BigDecimal;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainBitcoinyHTLCStatus {

View File

@ -1,11 +1,11 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainBuildRequest {

View File

@ -1,10 +1,10 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainCancelRequest {

View File

@ -1,14 +1,13 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.crosschain.AcctMode;
import org.qortal.data.crosschain.CrossChainTradeData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.qortal.crosschain.AcctMode;
import org.qortal.data.crosschain.CrossChainTradeData;
import io.swagger.v3.oas.annotations.media.Schema;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainOfferSummary {

View File

@ -1,10 +1,10 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainSecretRequest {

View File

@ -1,10 +1,10 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainTradeRequest {

View File

@ -1,13 +1,12 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.data.crosschain.CrossChainTradeData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.qortal.data.crosschain.CrossChainTradeData;
import io.swagger.v3.oas.annotations.media.Schema;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainTradeSummary {

View File

@ -0,0 +1,16 @@
package org.qortal.api.model;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class FileProperties {
public String filename;
public String mimeType;
public Long size;
public FileProperties() {
}
}

View File

@ -1,12 +1,11 @@
package org.qortal.api.model;
import java.util.List;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import io.swagger.v3.oas.annotations.media.Schema;
import java.util.List;
@Schema(description = "Group info, maybe including members")
// All properties to be converted to JSON via JAX-RS

View File

@ -1,11 +1,11 @@
package org.qortal.api.model;
import org.qortal.data.naming.NameData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import org.qortal.data.naming.NameData;
@XmlAccessorType(XmlAccessType.NONE)
public class NameSummary {

View File

@ -12,6 +12,7 @@ public class NodeInfo {
public long buildTimestamp;
public String nodeId;
public boolean isTestNet;
public String type;
public NodeInfo() {
}

View File

@ -1,11 +1,13 @@
package org.qortal.api.model;
import org.qortal.controller.Controller;
import org.qortal.controller.OnlineAccountsManager;
import org.qortal.controller.Synchronizer;
import org.qortal.network.Network;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import org.qortal.controller.Controller;
import org.qortal.network.Network;
@XmlAccessorType(XmlAccessType.FIELD)
public class NodeStatus {
@ -20,12 +22,12 @@ public class NodeStatus {
public final int height;
public NodeStatus() {
this.isMintingPossible = Controller.getInstance().isMintingPossible();
this.isMintingPossible = OnlineAccountsManager.getInstance().hasActiveOnlineAccountSignatures();
this.syncPercent = Controller.getInstance().getSyncPercent();
this.isSynchronizing = this.syncPercent != null;
this.syncPercent = Synchronizer.getInstance().getSyncPercent();
this.isSynchronizing = Synchronizer.getInstance().isSynchronizing();
this.numberOfConnections = Network.getInstance().getHandshakedPeers().size();
this.numberOfConnections = Network.getInstance().getImmutableHandshakedPeers().size();
this.height = Controller.getInstance().getChainHeight();
}

View File

@ -0,0 +1,15 @@
package org.qortal.api.model;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class PeersSummary {
public int inboundConnections;
public int outboundConnections;
public PeersSummary() {
}
}

View File

@ -0,0 +1,83 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.data.voting.VoteOnPollData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import java.util.List;
@Schema(description = "Poll vote info, including voters")
// All properties to be converted to JSON via JAX-RS
@XmlAccessorType(XmlAccessType.FIELD)
public class PollVotes {
@Schema(description = "List of individual votes")
@XmlElement(name = "votes")
public List<VoteOnPollData> votes;
@Schema(description = "Total number of votes")
public Integer totalVotes;
@Schema(description = "Total weight of votes")
public Integer totalWeight;
@Schema(description = "List of vote counts for each option")
public List<OptionCount> voteCounts;
@Schema(description = "List of vote weights for each option")
public List<OptionWeight> voteWeights;
// For JAX-RS
protected PollVotes() {
}
public PollVotes(List<VoteOnPollData> votes, Integer totalVotes, Integer totalWeight, List<OptionCount> voteCounts, List<OptionWeight> voteWeights) {
this.votes = votes;
this.totalVotes = totalVotes;
this.totalWeight = totalWeight;
this.voteCounts = voteCounts;
this.voteWeights = voteWeights;
}
@Schema(description = "Vote info")
// All properties to be converted to JSON via JAX-RS
@XmlAccessorType(XmlAccessType.FIELD)
public static class OptionCount {
@Schema(description = "Option name")
public String optionName;
@Schema(description = "Vote count")
public Integer voteCount;
// For JAX-RS
protected OptionCount() {
}
public OptionCount(String optionName, Integer voteCount) {
this.optionName = optionName;
this.voteCount = voteCount;
}
}
@Schema(description = "Vote weights")
// All properties to be converted to JSON via JAX-RS
@XmlAccessorType(XmlAccessType.FIELD)
public static class OptionWeight {
@Schema(description = "Option name")
public String optionName;
@Schema(description = "Vote weight")
public Integer voteWeight;
// For JAX-RS
protected OptionWeight() {
}
public OptionWeight(String optionName, Integer voteWeight) {
this.optionName = optionName;
this.voteWeight = voteWeight;
}
}
}

View File

@ -1,10 +1,10 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class RewardShareKeyRequest {

View File

@ -1,13 +1,12 @@
package org.qortal.api.model;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class SimpleForeignTransaction {

View File

@ -1,10 +1,10 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class SimpleTransactionSignRequest {

View File

@ -1,14 +1,13 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.data.asset.OrderData;
import org.qortal.data.asset.TradeData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import org.qortal.data.asset.OrderData;
import org.qortal.data.asset.TradeData;
import io.swagger.v3.oas.annotations.media.Schema;
@Schema(description = "Asset trade, including order info")
// All properties to be converted to JSON via JAX-RS
@XmlAccessorType(XmlAccessType.FIELD)

View File

@ -0,0 +1,17 @@
package org.qortal.api.model.crosschain;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class AddressRequest {
@Schema(description = "Litecoin BIP32 extended public key", example = "tpub___________________________________________________________________________________________________________")
public String xpub58;
public AddressRequest() {
}
}

View File

@ -1,11 +1,11 @@
package org.qortal.api.model.crosschain;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class BitcoinSendRequest {

View File

@ -0,0 +1,29 @@
package org.qortal.api.model.crosschain;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
@XmlAccessorType(XmlAccessType.FIELD)
public class DigibyteSendRequest {
@Schema(description = "Digibyte BIP32 extended private key", example = "tprv___________________________________________________________________________________________________________")
public String xprv58;
@Schema(description = "Recipient's Digibyte address ('legacy' P2PKH only)", example = "1DigByteEaterAddressDontSendf59kuE")
public String receivingAddress;
@Schema(description = "Amount of DGB to send", type = "number")
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
public long digibyteAmount;
@Schema(description = "Transaction fee per byte (optional). Default is 0.00000100 DGB (100 sats) per byte", example = "0.00000100", type = "number")
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
public Long feePerByte;
public DigibyteSendRequest() {
}
}

View File

@ -1,11 +1,11 @@
package org.qortal.api.model.crosschain;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class LitecoinSendRequest {

View File

@ -0,0 +1,32 @@
package org.qortal.api.model.crosschain;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
@XmlAccessorType(XmlAccessType.FIELD)
public class PirateChainSendRequest {
@Schema(description = "32 bytes of entropy, Base58 encoded", example = "5oSXF53qENtdUyKhqSxYzP57m6RhVFP9BJKRr9E5kRGV")
public String entropy58;
@Schema(description = "Recipient's Pirate Chain address", example = "zc...")
public String receivingAddress;
@Schema(description = "Amount of ARRR to send", type = "number")
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
public long arrrAmount;
@Schema(description = "Transaction fee per byte (optional). Default is 0.00000100 ARRR (100 sats) per byte", example = "0.00000100", type = "number")
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
public Long feePerByte;
@Schema(description = "Optional memo to include information for the recipient", example = "zc...")
public String memo;
public PirateChainSendRequest() {
}
}

View File

@ -0,0 +1,29 @@
package org.qortal.api.model.crosschain;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
@XmlAccessorType(XmlAccessType.FIELD)
public class RavencoinSendRequest {
@Schema(description = "Ravencoin BIP32 extended private key", example = "tprv___________________________________________________________________________________________________________")
public String xprv58;
@Schema(description = "Recipient's Ravencoin address ('legacy' P2PKH only)", example = "1RvnCoinEaterAddressDontSendf59kuE")
public String receivingAddress;
@Schema(description = "Amount of RVN to send", type = "number")
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
public long ravencoinAmount;
@Schema(description = "Transaction fee per byte (optional). Default is 0.00000100 RVN (100 sats) per byte", example = "0.00000100", type = "number")
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
public Long feePerByte;
public RavencoinSendRequest() {
}
}

View File

@ -1,13 +1,12 @@
package org.qortal.api.model.crosschain;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.crosschain.SupportedBlockchain;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.qortal.crosschain.SupportedBlockchain;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class TradeBotCreateRequest {

View File

@ -1,10 +1,10 @@
package org.qortal.api.model.crosschain;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class TradeBotRespondRequest {

View File

@ -0,0 +1,164 @@
package org.qortal.api.proxy.resource;
import org.qortal.api.ApiError;
import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.HTMLParser;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.DevProxyManager;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.core.Context;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.ProtocolException;
import java.net.URL;
import java.util.Enumeration;
@Path("/")
public class DevProxyServerResource {
@Context HttpServletRequest request;
@Context HttpServletResponse response;
@Context ServletContext context;
@GET
public HttpServletResponse getProxyIndex() {
return this.proxy("/");
}
@GET
@Path("{path:.*}")
public HttpServletResponse getProxyPath(@PathParam("path") String inPath) {
return this.proxy(inPath);
}
private HttpServletResponse proxy(String inPath) {
try {
String source = DevProxyManager.getInstance().getSourceHostAndPort();
if (!inPath.startsWith("/")) {
inPath = "/" + inPath;
}
String queryString = request.getQueryString() != null ? "?" + request.getQueryString() : "";
// Open URL
URL url = new URL(String.format("http://%s%s%s", source, inPath, queryString));
HttpURLConnection con = (HttpURLConnection) url.openConnection();
// Proxy the request data
this.proxyRequestToConnection(request, con);
try {
// Make the request and proxy the response code
response.setStatus(con.getResponseCode());
}
catch (ConnectException e) {
// Tey converting localhost / 127.0.0.1 to IPv6 [::1]
if (source.startsWith("localhost") || source.startsWith("127.0.0.1")) {
int port = 80;
String[] parts = source.split(":");
if (parts.length > 1) {
port = Integer.parseInt(parts[1]);
}
source = String.format("[::1]:%d", port);
}
// Retry connection
url = new URL(String.format("http://%s%s%s", source, inPath, queryString));
con = (HttpURLConnection) url.openConnection();
this.proxyRequestToConnection(request, con);
response.setStatus(con.getResponseCode());
}
// Proxy the response data back to the caller
this.proxyConnectionToResponse(con, response, inPath);
} catch (IOException e) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, e.getMessage());
}
return response;
}
private void proxyRequestToConnection(HttpServletRequest request, HttpURLConnection con) throws ProtocolException {
// Proxy the request method
con.setRequestMethod(request.getMethod());
// Proxy the request headers
Enumeration<String> headerNames = request.getHeaderNames();
while (headerNames.hasMoreElements()) {
String headerName = headerNames.nextElement();
String headerValue = request.getHeader(headerName);
con.setRequestProperty(headerName, headerValue);
}
// TODO: proxy any POST parameters from "request" to "con"
}
private void proxyConnectionToResponse(HttpURLConnection con, HttpServletResponse response, String inPath) throws IOException {
// Proxy the response headers
for (int i = 0; ; i++) {
String headerKey = con.getHeaderFieldKey(i);
String headerValue = con.getHeaderField(i);
if (headerKey != null && headerValue != null) {
response.addHeader(headerKey, headerValue);
continue;
}
break;
}
// Read the response body
InputStream inputStream = con.getInputStream();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
byte[] data = outputStream.toByteArray(); // TODO: limit file size that can be read into memory
// Close the streams
outputStream.close();
inputStream.close();
// Extract filename
String filename = "";
if (inPath.contains("/")) {
String[] parts = inPath.split("/");
if (parts.length > 0) {
filename = parts[parts.length - 1];
}
}
// Parse and modify output if needed
if (HTMLParser.isHtmlFile(filename)) {
// HTML file - needs to be parsed
HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, "light", true);
htmlParser.addAdditionalHeaderTags();
response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' ws:; font-src 'self' data:;");
response.setContentType(con.getContentType());
response.setContentLength(htmlParser.getData().length);
response.getOutputStream().write(htmlParser.getData());
}
else {
// Regular file - can be streamed directly
response.addHeader("Content-Security-Policy", "default-src 'self'");
response.setContentType(con.getContentType());
response.setContentLength(data.length);
response.getOutputStream().write(data);
}
}
}

View File

@ -1,5 +1,6 @@
package org.qortal.api.resource;
import com.google.common.primitives.Bytes;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.ArraySchema;
@ -9,34 +10,18 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import io.swagger.v3.oas.annotations.tags.Tag;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import org.qortal.account.Account;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.api.ApiError;
import org.qortal.api.ApiErrors;
import org.qortal.api.ApiException;
import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.Security;
import org.qortal.api.*;
import org.qortal.api.model.AccountPenaltyStats;
import org.qortal.api.model.ApiOnlineAccount;
import org.qortal.api.model.RewardShareKeyRequest;
import org.qortal.asset.Asset;
import org.qortal.controller.Controller;
import org.qortal.controller.LiteNode;
import org.qortal.controller.OnlineAccountsManager;
import org.qortal.crypto.Crypto;
import org.qortal.data.account.AccountData;
import org.qortal.data.account.AccountPenaltyData;
import org.qortal.data.account.RewardShareData;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.data.network.OnlineAccountLevel;
@ -59,7 +44,15 @@ import org.qortal.transform.transaction.TransactionTransformer;
import org.qortal.utils.Amounts;
import org.qortal.utils.Base58;
import com.google.common.primitives.Bytes;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
@Path("/addresses")
@Tag(name = "Addresses")
@ -113,18 +106,26 @@ public class AddressesResource {
if (!Crypto.isValidAddress(address))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
byte[] lastReference = null;
AccountData accountData;
try (final Repository repository = RepositoryManager.getRepository()) {
AccountData accountData = repository.getAccountRepository().getAccount(address);
// Not found?
if (accountData == null)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN);
lastReference = accountData.getReference();
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
if (Settings.getInstance().isLite()) {
// Lite nodes request data from peers instead of the local db
accountData = LiteNode.getInstance().fetchAccountData(address);
}
else {
// All other node types request data from local db
try (final Repository repository = RepositoryManager.getRepository()) {
accountData = repository.getAccountRepository().getAccount(address);
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
// Not found?
if (accountData == null)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN);
byte[] lastReference = accountData.getReference();
if (lastReference == null || lastReference.length == 0)
return "false";
@ -160,7 +161,7 @@ public class AddressesResource {
)
@ApiErrors({ApiError.PUBLIC_KEY_NOT_FOUND, ApiError.REPOSITORY_ISSUE})
public List<ApiOnlineAccount> getOnlineAccounts() {
List<OnlineAccountData> onlineAccounts = Controller.getInstance().getOnlineAccounts();
List<OnlineAccountData> onlineAccounts = OnlineAccountsManager.getInstance().getOnlineAccounts();
// Map OnlineAccountData entries to OnlineAccount via reward-share data
try (final Repository repository = RepositoryManager.getRepository()) {
@ -195,14 +196,18 @@ public class AddressesResource {
)
@ApiErrors({ApiError.PUBLIC_KEY_NOT_FOUND, ApiError.REPOSITORY_ISSUE})
public List<OnlineAccountLevel> getOnlineAccountsByLevel() {
List<OnlineAccountData> onlineAccounts = Controller.getInstance().getOnlineAccounts();
List<OnlineAccountData> onlineAccounts = OnlineAccountsManager.getInstance().getOnlineAccounts();
try (final Repository repository = RepositoryManager.getRepository()) {
List<OnlineAccountLevel> onlineAccountLevels = new ArrayList<>();
// Prepopulate all levels
for (int i=0; i<=10; i++)
onlineAccountLevels.add(new OnlineAccountLevel(i, 0));
for (OnlineAccountData onlineAccountData : onlineAccounts) {
try {
final int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, onlineAccountData.getPublicKey());
final int minterLevel = Account.getRewardShareEffectiveMintingLevelIncludingLevelZero(repository, onlineAccountData.getPublicKey());
OnlineAccountLevel onlineAccountLevel = onlineAccountLevels.stream()
.filter(a -> a.getLevel() == minterLevel)
@ -228,8 +233,7 @@ public class AddressesResource {
}
} catch (DataException e) {
continue;
}
}
}
// Sort by level
@ -462,6 +466,54 @@ public class AddressesResource {
}
}
@GET
@Path("/penalties")
@Operation(
summary = "Get addresses with penalties",
description = "Returns a list of accounts with a blocksMintedPenalty",
responses = {
@ApiResponse(
description = "accounts with penalties",
content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = AccountPenaltyData.class)))
)
}
)
@ApiErrors({ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE})
public List<AccountPenaltyData> getAccountsWithPenalties() {
try (final Repository repository = RepositoryManager.getRepository()) {
List<AccountData> accounts = repository.getAccountRepository().getPenaltyAccounts();
List<AccountPenaltyData> penalties = accounts.stream().map(a -> new AccountPenaltyData(a.getAddress(), a.getBlocksMintedPenalty())).collect(Collectors.toList());
return penalties;
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
@GET
@Path("/penalties/stats")
@Operation(
summary = "Get stats about current penalties",
responses = {
@ApiResponse(
description = "aggregated stats about accounts with penalties",
content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = AccountPenaltyStats.class)))
)
}
)
@ApiErrors({ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE})
public AccountPenaltyStats getPenaltyStats() {
try (final Repository repository = RepositoryManager.getRepository()) {
List<AccountData> accounts = repository.getAccountRepository().getPenaltyAccounts();
return AccountPenaltyStats.fromAccounts(accounts);
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
@POST
@Path("/publicize")
@Operation(
@ -537,7 +589,7 @@ public class AddressesResource {
)
@ApiErrors({ApiError.TRANSACTION_INVALID, ApiError.INVALID_DATA, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE})
@SecurityRequirement(name = "apiKey")
public String computePublicize(String rawBytes58) {
public String computePublicize(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String rawBytes58) {
Security.checkApiCallAllowed(request);
try (final Repository repository = RepositoryManager.getRepository()) {

View File

@ -13,12 +13,6 @@ import io.swagger.v3.oas.models.media.MediaType;
import io.swagger.v3.oas.models.media.Schema;
import io.swagger.v3.oas.models.parameters.Parameter;
import io.swagger.v3.oas.models.responses.ApiResponse;
import java.lang.reflect.Method;
import java.util.Locale;
import javax.ws.rs.Path;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.ApiError;
@ -27,6 +21,10 @@ import org.qortal.api.ApiErrors;
import org.qortal.api.ApiService;
import org.qortal.globalization.Translator;
import javax.ws.rs.Path;
import java.lang.reflect.Method;
import java.util.Locale;
public class AnnotationPostProcessor implements ReaderListener {
private static final Logger LOGGER = LogManager.getLogger(AnnotationPostProcessor.class);

Some files were not shown because too many files have changed in this diff Show More