AlphaX
/
qortal
forked from Qortal/qortal
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Rework of file list requests and relays, allowing it to handle multiple chunk resources in a much more sensible way. 

This could create a lot of additional relay traffic as a result, so needs lots of testing and possibly optimizing.
trade-portal-updates
CalDescent 3 years ago
parent
8fb2d38cd1
commit
6f80a6c08a
2 changed files with 81 additions and 34 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 91
      src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
  2. 24
      src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java

91
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
Unescape View File

@ -291,7 +291,20 @@ public class ArbitraryDataFileListManager {
return true;
}
public void deleteFileListRequestsForSignature(byte[] signature) {
String signature58 = Base58.encode(signature);
for (Iterator<Map.Entry<Integer, Triple<String, Peer, Long>>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) {
Map.Entry<Integer, Triple<String, Peer, Long>> entry = it.next();
if (entry == null || entry.getKey() == null || entry.getValue() != null) {
continue;
}
if (Objects.equals(entry.getValue().getA(), signature58)) {
// Update requests map to reflect that we've received all chunks
Triple<String, Peer, Long> newEntry = new Triple<>(null, null, entry.getValue().getC());
arbitraryDataFileListRequests.put(entry.getKey(), newEntry);
}
}
}
// Network handlers
@ -304,7 +317,7 @@ public class ArbitraryDataFileListManager {
ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message;
LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size());
// Do we have a pending request for this data? // TODO: might we want to relay all of them anyway?
// Do we have a pending request for this data?
Triple<String, Peer, Long> request = arbitraryDataFileListRequests.get(message.getId());
if (request == null || request.getA() == null) {
return;
@ -350,10 +363,6 @@ public class ArbitraryDataFileListManager {
// }
// }
// Update requests map to reflect that we've received it
Triple<String, Peer, Long> newEntry = new Triple<>(null, null, request.getC());
arbitraryDataFileListRequests.put(message.getId(), newEntry);
if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) {
// Go and fetch the actual data, since this isn't a relay request
arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, hashes);
@ -412,6 +421,7 @@ public class ArbitraryDataFileListManager {
List<byte[]> hashes = new ArrayList<>();
ArbitraryTransactionData transactionData = null;
boolean allChunksExist = false;
try (final Repository repository = RepositoryManager.getRepository()) {
@ -435,6 +445,9 @@ public class ArbitraryDataFileListManager {
hashes.add(arbitraryDataFile.getMetadataHash());
}
// Check if we have all the chunks (and the metadata file)
allChunksExist = arbitraryDataFile.allChunksExist();
for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) {
if (chunk.exists()) {
hashes.add(chunk.getHash());
@ -459,49 +472,59 @@ public class ArbitraryDataFileListManager {
// We should only respond if we have at least one hash
if (hashes.size() > 0) {
// Update requests map to reflect that we've sent it
newEntry = new Triple<>(signature58, null, now);
arbitraryDataFileListRequests.put(message.getId(), newEntry);
// We have all the chunks, so update requests map to reflect that we've sent it
// There is no need to keep track of the request, as we can serve all the chunks
if (allChunksExist) {
newEntry = new Triple<>(null, null, now);
arbitraryDataFileListRequests.put(message.getId(), newEntry);
}
ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
arbitraryDataFileListMessage.setId(message.getId());
if (!peer.sendMessage(arbitraryDataFileListMessage)) {
LOGGER.debug("Couldn't send list of hashes");
peer.disconnect("failed to send list of hashes");
return;
}
LOGGER.debug("Sent list of hashes (count: {})", hashes.size());
if (allChunksExist) {
// Nothing left to do, so return to prevent any unnecessary forwarding from occurring
LOGGER.debug("No need for any forwarding because file list request is fully served");
return;
}
}
else {
boolean isBlocked = (transactionData == null || ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName()));
if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
// In relay mode - so ask our other peers if they have it
long requestTime = getArbitraryDataFileListMessage.getRequestTime();
int requestHops = getArbitraryDataFileListMessage.getRequestHops();
getArbitraryDataFileListMessage.setRequestHops(++requestHops);
long totalRequestTime = now - requestTime;
if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) {
// Relay request hasn't timed out yet, so can potentially be rebroadcast
if (requestHops < RELAY_REQUEST_MAX_HOPS) {
// Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast
LOGGER.info("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops);
Network.getInstance().broadcast(
broadcastPeer -> broadcastPeer == peer ||
Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost())
? null : getArbitraryDataFileListMessage);
}
else {
// This relay request has reached the maximum number of allowed hops
}
// We may need to forward this request on
boolean isBlocked = (transactionData == null || ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName()));
if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
// In relay mode - so ask our other peers if they have it
long requestTime = getArbitraryDataFileListMessage.getRequestTime();
int requestHops = getArbitraryDataFileListMessage.getRequestHops();
getArbitraryDataFileListMessage.setRequestHops(++requestHops);
long totalRequestTime = now - requestTime;
if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) {
// Relay request hasn't timed out yet, so can potentially be rebroadcast
if (requestHops < RELAY_REQUEST_MAX_HOPS) {
// Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast
LOGGER.info("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops);
Network.getInstance().broadcast(
broadcastPeer -> broadcastPeer == peer ||
Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost())
? null : getArbitraryDataFileListMessage);
}
else {
// This relay request has timed out
// This relay request has reached the maximum number of allowed hops
}
}
else {
// This relay request has timed out
}
}
}

24
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java
Unescape View File

@ -15,6 +15,7 @@ import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.utils.ArbitraryTransactionUtils;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
import org.qortal.utils.Triple;
@ -187,6 +188,9 @@ public class ArbitraryDataFileManager {
arbitraryDataFileRequests.remove(hash58);
LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58));
// We may need to remove the file list request, if we have all the files for this transaction
this.handleFileListRequests(signature);
if (message == null || message.getType() != Message.MessageType.ARBITRARY_DATA_FILE) {
return null;
}
@ -209,6 +213,26 @@ public class ArbitraryDataFileManager {
return arbitraryDataFileMessage;
}
private void handleFileListRequests(byte[] signature) {
try (final Repository repository = RepositoryManager.getRepository()) {
// Fetch the transaction data
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
if (arbitraryTransactionData == null) {
return;
}
boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData);
if (allChunksExist) {
// Update requests map to reflect that we've received all chunks
ArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature);
}
} catch (DataException e) {
LOGGER.debug("Unable to handle file list requests: {}", e.getMessage());
}
}
public void handleArbitraryDataFileForwarding(Peer requestingPeer, Message message, Message originalMessage) {
// Return if there is no originally requesting peer to forward to

Write Preview
Loading…
Cancel
Save