Qortal/qortal
1
0
Fork 0
mirror of https://github.com/Qortal/qortal.git synced 2025-07-19 19:01:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #8 from Philreact/fix/load-data

Browse Source 
fix issue of not breaking when file is complete
This commit is contained in:
kennycud 2025-05-28 17:21:55 -07:00 committed by GitHub
commit fa8b9f2cee
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 111 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -1902,7 +1902,6 @@ public String finalizeUpload(
private void download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts, boolean attachment, String attachmentFilename) { private void download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts, boolean attachment, String attachmentFilename) {
try { try {
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
int attempts = 0; int attempts = 0;
@ -1928,7 +1927,6 @@ public String finalizeUpload(
} }
} }
} }
Thread.sleep(3000L);
} }
} }
@ -1981,54 +1979,58 @@ public String finalizeUpload(
// 3. Set Content-Disposition header // 3. Set Content-Disposition header
response.setHeader("Content-Disposition", "attachment; filename=\"" + rawFilename + "\""); response.setHeader("Content-Disposition", "attachment; filename=\"" + rawFilename + "\"");
} }
// Determine the total size of the requested file // Determine the total size of the requested file
long fileSize = Files.size(path); long fileSize = Files.size(path);
String mimeType = context.getMimeType(path.toString()); String mimeType = context.getMimeType(path.toString());
// Attempt to read the "Range" header from the request to support partial content delivery (e.g., for video streaming or resumable downloads) // Attempt to read the "Range" header from the request to support partial content delivery (e.g., for video streaming or resumable downloads)
String range = request.getHeader("Range"); String range = request.getHeader("Range");
long rangeStart = 0; long rangeStart = 0;
long rangeEnd = fileSize - 1; long rangeEnd = fileSize - 1;
boolean isPartial = false; boolean isPartial = false;
// If a Range header is present and no base64 encoding is requested, parse the range values // If a Range header is present and no base64 encoding is requested, parse the range values
if (range != null && encoding == null) { if (range != null && encoding == null) {
range = range.replace("bytes=", ""); // Remove the "bytes=" prefix range = range.replace("bytes=", ""); // Remove the "bytes=" prefix
String[] parts = range.split("-"); // Split the range into start and end String[] parts = range.split("-"); // Split the range into start and end
// Parse range start // Parse range start
if (parts.length > 0 && !parts[0].isEmpty()) { if (parts.length > 0 && !parts[0].isEmpty()) {
rangeStart = Long.parseLong(parts[0]); rangeStart = Long.parseLong(parts[0]);
}
// Parse range end, if present
if (parts.length > 1 && !parts[1].isEmpty()) {
rangeEnd = Long.parseLong(parts[1]);
}
isPartial = true; // Indicate that this is a partial content request
} }
// Calculate how many bytes should be sent in the response // Parse range end, if present
long contentLength = rangeEnd - rangeStart + 1; if (parts.length > 1 && !parts[1].isEmpty()) {
rangeEnd = Long.parseLong(parts[1]);
// Inform the client that byte ranges are supported
response.setHeader("Accept-Ranges", "bytes");
if (isPartial) {
// If partial content was requested, return 206 Partial Content with appropriate headers
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize));
} else {
// Otherwise, return the entire file with status 200 OK
response.setStatus(HttpServletResponse.SC_OK);
} }
// Initialize output streams for writing the file to the response isPartial = true; // Indicate that this is a partial content request
OutputStream rawOut = response.getOutputStream(); }
OutputStream base64Out = null;
OutputStream gzipOut = null; // Calculate how many bytes should be sent in the response
long contentLength = rangeEnd - rangeStart + 1;
// Inform the client that byte ranges are supported
response.setHeader("Accept-Ranges", "bytes");
if (isPartial) {
// If partial content was requested, return 206 Partial Content with appropriate headers
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize));
} else {
// Otherwise, return the entire file with status 200 OK
response.setStatus(HttpServletResponse.SC_OK);
}
// Initialize output streams for writing the file to the response
OutputStream rawOut = null;
OutputStream base64Out = null;
OutputStream gzipOut = null;
try {
rawOut = response.getOutputStream();
if (encoding != null && "base64".equalsIgnoreCase(encoding)) { if (encoding != null && "base64".equalsIgnoreCase(encoding)) {
// If base64 encoding is requested, override content type // If base64 encoding is requested, override content type
@ -2054,45 +2056,57 @@ public String finalizeUpload(
response.setContentType(mimeType != null ? mimeType : "application/octet-stream"); response.setContentType(mimeType != null ? mimeType : "application/octet-stream");
response.setContentLength((int) contentLength); response.setContentLength((int) contentLength);
} }
// Stream file content
try (InputStream inputStream = Files.newInputStream(path)) { // Stream file content
if (rangeStart > 0) { try (InputStream inputStream = Files.newInputStream(path)) {
inputStream.skip(rangeStart); if (rangeStart > 0) {
inputStream.skip(rangeStart);
}
byte[] buffer = new byte[65536];
long bytesRemaining = contentLength;
int bytesRead;
while (bytesRemaining > 0 && (bytesRead = inputStream.read(buffer, 0, (int) Math.min(buffer.length, bytesRemaining))) != -1) {
rawOut.write(buffer, 0, bytesRead);
bytesRemaining -= bytesRead;
}
} }
byte[] buffer = new byte[65536]; // Stream finished
long bytesRemaining = contentLength; if (base64Out != null) {
int bytesRead; base64Out.close(); // Also flushes and closes the wrapped gzipOut
} else if (gzipOut != null) {
while (bytesRemaining > 0 && (bytesRead = inputStream.read(buffer, 0, (int) Math.min(buffer.length, bytesRemaining))) != -1) { gzipOut.close(); // Only close gzipOut if it wasn't wrapped by base64Out
rawOut.write(buffer, 0, bytesRead); } else {
bytesRemaining -= bytesRead; rawOut.flush(); // Flush only the base output stream if nothing was wrapped
} }
if (!response.isCommitted()) {
response.setStatus(HttpServletResponse.SC_OK);
response.getWriter().write(" ");
}
} catch (IOException e) {
// Streaming errors should not rethrow — just log
LOGGER.warn(String.format("Streaming error for %s %s: %s", service, name, e.getMessage()), e);
} }
// Stream finished
if (base64Out != null) {
base64Out.close(); // Also flushes and closes the wrapped gzipOut
} else if (gzipOut != null) {
gzipOut.close(); // Only close gzipOut if it wasn't wrapped by base64Out
} else {
rawOut.flush(); // Flush only the base output stream if nothing was wrapped
}
if (!response.isCommitted()) {
response.setStatus(HttpServletResponse.SC_OK);
response.getWriter().write(" ");
}
} catch (IOException | InterruptedException | ApiException | DataException e) { } catch (IOException | ApiException | DataException e) {
LOGGER.error(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e); LOGGER.warn(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e);
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); if (!response.isCommitted()) {
} throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage());
catch ( NumberFormatException e) { }
LOGGER.error(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e); } catch (NumberFormatException e) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage()); LOGGER.warn(String.format("Invalid range for %s %s: %s", service, name, e.getMessage()), e);
if (!response.isCommitted()) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage());
}
} }
} }
private FileProperties getFileProperties(Service service, String name, String identifier) { private FileProperties getFileProperties(Service service, String name, String identifier) {
try { try {
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);

2
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
View File

@ -400,7 +400,7 @@ public class ArbitraryDataFileListManager {
String signature58 = Base58.encode(signature); String signature58 = Base58.encode(signature);
for (Iterator<Map.Entry<Integer, Triple<String, Peer, Long>>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) { for (Iterator<Map.Entry<Integer, Triple<String, Peer, Long>>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) {
Map.Entry<Integer, Triple<String, Peer, Long>> entry = it.next(); Map.Entry<Integer, Triple<String, Peer, Long>> entry = it.next();
if (entry == null || entry.getKey() == null || entry.getValue() != null) { if (entry == null || entry.getKey() == null || entry.getValue() == null) {
continue; continue;
} }
if (Objects.equals(entry.getValue().getA(), signature58)) { if (Objects.equals(entry.getValue().getA(), signature58)) {

14
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java
View File

@ -212,8 +212,7 @@ public class ArbitraryDataFileManager extends Thread {
arbitraryDataFileRequests.remove(hash58); arbitraryDataFileRequests.remove(hash58);
LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58)); LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58));
// We may need to remove the file list request, if we have all the files for this transaction
this.handleFileListRequests(signature);
if (response == null) { if (response == null) {
LOGGER.debug("Received null response from peer {}", peer); LOGGER.debug("Received null response from peer {}", peer);
@ -258,6 +257,9 @@ public class ArbitraryDataFileManager extends Thread {
} }
} }
// We may need to remove the file list request, if we have all the files for this transaction
this.handleFileListRequests(signature);
return arbitraryDataFile; return arbitraryDataFile;
} }
@ -270,10 +272,12 @@ public class ArbitraryDataFileManager extends Thread {
return; return;
} }
boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData); boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData);
if (completeFileExists) {
String signature58 = Base58.encode(arbitraryTransactionData.getSignature());
LOGGER.info("All chunks or complete file exist for transaction {}", signature58);
if (allChunksExist) {
// Update requests map to reflect that we've received all chunks
ArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature); ArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature);
} }