Qortal/qortal
1
0
Fork 0
mirror of https://github.com/Qortal/qortal.git synced 2025-07-19 19:01:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7 from Philreact/feature/allow-for-unlimited-size-publishes

Browse Source 
Feature/allow for unlimited size publishes
This commit is contained in:
kennycud 2025-05-25 11:45:27 -07:00 committed by GitHub
commit 5013c68b61
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 700 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pom.xml
View File

@ -796,5 +796,10 @@
<artifactId>jaxb-runtime</artifactId>
<version>${jaxb-runtime.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<version>3.1.0</version>
</dependency>
</dependencies>
</project>

1
src/main/java/org/qortal/api/ApiService.java
View File

@ -46,6 +46,7 @@ public class ApiService {
private ApiService() {
this.config = new ResourceConfig();
this.config.packages("org.qortal.api.resource", "org.qortal.api.restricted.resource");
this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
this.config.register(OpenApiResource.class);
this.config.register(ApiDefinition.class);
this.config.register(AnnotationPostProcessor.class);

1
src/main/java/org/qortal/api/DevProxyService.java
View File

@ -40,6 +40,7 @@ public class DevProxyService {
private DevProxyService() {
this.config = new ResourceConfig();
this.config.packages("org.qortal.api.proxy.resource", "org.qortal.api.resource");
this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
this.config.register(OpenApiResource.class);
this.config.register(ApiDefinition.class);
this.config.register(AnnotationPostProcessor.class);

1
src/main/java/org/qortal/api/DomainMapService.java
View File

@ -39,6 +39,7 @@ public class DomainMapService {
private DomainMapService() {
this.config = new ResourceConfig();
this.config.packages("org.qortal.api.resource", "org.qortal.api.domainmap.resource");
this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
this.config.register(OpenApiResource.class);
this.config.register(ApiDefinition.class);
this.config.register(AnnotationPostProcessor.class);

1
src/main/java/org/qortal/api/GatewayService.java
View File

@ -39,6 +39,7 @@ public class GatewayService {
private GatewayService() {
this.config = new ResourceConfig();
this.config.packages("org.qortal.api.resource", "org.qortal.api.gateway.resource");
this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
this.config.register(OpenApiResource.class);
this.config.register(ApiDefinition.class);
this.config.register(AnnotationPostProcessor.class);

679
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -3,6 +3,7 @@ package org.qortal.api.resource;
import com.google.common.primitives.Bytes;
import com.j256.simplemagic.ContentInfo;
import com.j256.simplemagic.ContentInfoUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.ArraySchema;
@ -12,6 +13,7 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.LogManager;
@ -63,14 +65,19 @@ import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.FileNameMap;
import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@ -78,6 +85,16 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.zip.GZIPOutputStream;
import org.apache.tika.Tika;
import org.apache.tika.mime.MimeTypeException;
import org.apache.tika.mime.MimeTypes;
import javax.ws.rs.core.Response;
import org.glassfish.jersey.media.multipart.FormDataParam;
import static org.qortal.api.ApiError.REPOSITORY_ISSUE;
@Path("/arbitrary")
@Tag(name = "Arbitrary")
@ -686,20 +703,20 @@ public class ArbitraryResource {
)
}
)
public HttpServletResponse get(@PathParam("service") Service service,
public void get(@PathParam("service") Service service,
@PathParam("name") String name,
@QueryParam("filepath") String filepath,
@QueryParam("encoding") String encoding,
@QueryParam("rebuild") boolean rebuild,
@QueryParam("async") boolean async,
@QueryParam("attempts") Integer attempts) {
@QueryParam("attempts") Integer attempts, @QueryParam("attachment") boolean attachment, @QueryParam("attachmentFilename") String attachmentFilename) {
// Authentication can be bypassed in the settings, for those running public QDN nodes
if (!Settings.getInstance().isQDNAuthBypassEnabled()) {
Security.checkApiCallAllowed(request);
}
return this.download(service, name, null, filepath, encoding, rebuild, async, attempts);
this.download(service, name, null, filepath, encoding, rebuild, async, attempts, attachment, attachmentFilename);
}
@GET
@ -719,21 +736,21 @@ public class ArbitraryResource {
)
}
)
public HttpServletResponse get(@PathParam("service") Service service,
public void get(@PathParam("service") Service service,
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@QueryParam("filepath") String filepath,
@QueryParam("encoding") String encoding,
@QueryParam("rebuild") boolean rebuild,
@QueryParam("async") boolean async,
@QueryParam("attempts") Integer attempts) {
@QueryParam("attempts") Integer attempts, @QueryParam("attachment") boolean attachment, @QueryParam("attachmentFilename") String attachmentFilename) {
// Authentication can be bypassed in the settings, for those running public QDN nodes
if (!Settings.getInstance().isQDNAuthBypassEnabled()) {
Security.checkApiCallAllowed(request, null);
}
return this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts);
this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts, attachment, attachmentFilename);
}
@ -878,6 +895,464 @@ public class ArbitraryResource {
}
@GET
@Path("/check/tmp")
@Produces(MediaType.TEXT_PLAIN)
@Operation(
summary = "Check if the disk has enough disk space for an upcoming upload",
responses = {
@ApiResponse(description = "OK if sufficient space", responseCode = "200"),
@ApiResponse(description = "Insufficient space", responseCode = "507") // 507 = Insufficient Storage
}
)
@SecurityRequirement(name = "apiKey")
public Response checkUploadSpace(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@QueryParam("totalSize") Long totalSize) {
Security.checkApiCallAllowed(request);
if (totalSize == null || totalSize <= 0) {
return Response.status(Response.Status.BAD_REQUEST)
.entity("Missing or invalid totalSize parameter").build();
}
File uploadDir = new File("uploads-temp");
if (!uploadDir.exists()) {
uploadDir.mkdirs(); // ensure the folder exists
}
long usableSpace = uploadDir.getUsableSpace();
long requiredSpace = (long)(((double)totalSize) * 2.2); // estimate for chunks + merge
if (usableSpace < requiredSpace) {
return Response.status(507).entity("Insufficient disk space").build();
}
return Response.ok("Sufficient disk space").build();
}
@POST
@Path("/{service}/{name}/chunk")
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Operation(
summary = "Upload a single file chunk to be later assembled into a complete arbitrary resource (no identifier)",
requestBody = @RequestBody(
required = true,
content = @Content(
mediaType = MediaType.MULTIPART_FORM_DATA,
schema = @Schema(
implementation = Object.class
)
)
),
responses = {
@ApiResponse(
description = "Chunk uploaded successfully",
responseCode = "200"
),
@ApiResponse(
description = "Error writing chunk",
responseCode = "500"
)
}
)
@SecurityRequirement(name = "apiKey")
public Response uploadChunkNoIdentifier(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") String serviceString,
@PathParam("name") String name,
@FormDataParam("chunk") InputStream chunkStream,
@FormDataParam("index") int index) {
Security.checkApiCallAllowed(request);
try {
String safeService = Paths.get(serviceString).getFileName().toString();
String safeName = Paths.get(name).getFileName().toString();
java.nio.file.Path tempDir = Paths.get("uploads-temp", safeService, safeName);
Files.createDirectories(tempDir);
java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index);
Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING);
return Response.ok("Chunk " + index + " received").build();
} catch (IOException e) {
LOGGER.error("Failed to write chunk {} for service '{}' and name '{}'", index, serviceString, name, e);
return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build();
}
}
@POST
@Path("/{service}/{name}/finalize")
@Produces(MediaType.TEXT_PLAIN)
@Operation(
summary = "Finalize a chunked upload (no identifier) and build a raw, unsigned, ARBITRARY transaction",
responses = {
@ApiResponse(
description = "raw, unsigned, ARBITRARY transaction encoded in Base58",
content = @Content(mediaType = MediaType.TEXT_PLAIN)
)
}
)
@SecurityRequirement(name = "apiKey")
public String finalizeUploadNoIdentifier(
@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") String serviceString,
@PathParam("name") String name,
@QueryParam("title") String title,
@QueryParam("description") String description,
@QueryParam("tags") List<String> tags,
@QueryParam("category") Category category,
@QueryParam("filename") String filename,
@QueryParam("fee") Long fee,
@QueryParam("preview") Boolean preview,
@QueryParam("isZip") Boolean isZip
) {
Security.checkApiCallAllowed(request);
java.nio.file.Path tempFile = null;
java.nio.file.Path tempDir = null;
java.nio.file.Path chunkDir = null;
String safeService = Paths.get(serviceString).getFileName().toString();
String safeName = Paths.get(name).getFileName().toString();
try {
chunkDir = Paths.get("uploads-temp", safeService, safeName);
if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload");
}
String safeFilename = (filename == null || filename.isBlank()) ? "qortal-" + NTP.getTime() : filename;
tempDir = Files.createTempDirectory("qortal-");
String sanitizedFilename = Paths.get(safeFilename).getFileName().toString();
tempFile = tempDir.resolve(sanitizedFilename);
try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
byte[] buffer = new byte[65536];
for (java.nio.file.Path chunk : Files.list(chunkDir)
.filter(path -> path.getFileName().toString().startsWith("chunk_"))
.sorted(Comparator.comparingInt(path -> {
String name2 = path.getFileName().toString();
String numberPart = name2.substring("chunk_".length());
return Integer.parseInt(numberPart);
})).collect(Collectors.toList())) {
try (InputStream in = Files.newInputStream(chunk)) {
int bytesRead;
while ((bytesRead = in.read(buffer)) != -1) {
out.write(buffer, 0, bytesRead);
}
}
}
}
String detectedExtension = "";
String uploadFilename = null;
boolean extensionIsValid = false;
if (filename != null && !filename.isBlank()) {
int lastDot = filename.lastIndexOf('.');
if (lastDot > 0 && lastDot < filename.length() - 1) {
extensionIsValid = true;
uploadFilename = filename;
}
}
if (!extensionIsValid) {
Tika tika = new Tika();
String mimeType = tika.detect(tempFile.toFile());
try {
MimeTypes allTypes = MimeTypes.getDefaultMimeTypes();
org.apache.tika.mime.MimeType mime = allTypes.forName(mimeType);
detectedExtension = mime.getExtension();
} catch (MimeTypeException e) {
LOGGER.warn("Could not determine file extension for MIME type: {}", mimeType, e);
}
if (filename != null && !filename.isBlank()) {
int lastDot = filename.lastIndexOf('.');
String baseName = (lastDot > 0) ? filename.substring(0, lastDot) : filename;
uploadFilename = baseName + (detectedExtension != null ? detectedExtension : "");
} else {
uploadFilename = "qortal-" + NTP.getTime() + (detectedExtension != null ? detectedExtension : "");
}
}
Boolean isZipBoolean = false;
if (isZip != null && isZip) {
isZipBoolean = true;
}
// ✅ Call upload with `null` as identifier
return this.upload(
Service.valueOf(serviceString),
name,
null, // no identifier
tempFile.toString(),
null,
null,
isZipBoolean,
fee,
uploadFilename,
title,
description,
tags,
category,
preview
);
} catch (IOException e) {
LOGGER.error("Failed to merge chunks for service='{}', name='{}'", serviceString, name, e);
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage());
} finally {
if (tempDir != null) {
try {
Files.walk(tempDir)
.sorted(Comparator.reverseOrder())
.map(java.nio.file.Path::toFile)
.forEach(File::delete);
} catch (IOException e) {
LOGGER.warn("Failed to delete temp directory: {}", tempDir, e);
}
}
try {
Files.walk(chunkDir)
.sorted(Comparator.reverseOrder())
.map(java.nio.file.Path::toFile)
.forEach(File::delete);
} catch (IOException e) {
LOGGER.warn("Failed to delete chunk directory: {}", chunkDir, e);
}
}
}
@POST
@Path("/{service}/{name}/{identifier}/chunk")
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Operation(
summary = "Upload a single file chunk to be later assembled into a complete arbitrary resource",
requestBody = @RequestBody(
required = true,
content = @Content(
mediaType = MediaType.MULTIPART_FORM_DATA,
schema = @Schema(
implementation = Object.class
)
)
),
responses = {
@ApiResponse(
description = "Chunk uploaded successfully",
responseCode = "200"
),
@ApiResponse(
description = "Error writing chunk",
responseCode = "500"
)
}
)
@SecurityRequirement(name = "apiKey")
public Response uploadChunk(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") String serviceString,
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@FormDataParam("chunk") InputStream chunkStream,
@FormDataParam("index") int index) {
Security.checkApiCallAllowed(request);
try {
String safeService = Paths.get(serviceString).getFileName().toString();
String safeName = Paths.get(name).getFileName().toString();
String safeIdentifier = Paths.get(identifier).getFileName().toString();
java.nio.file.Path tempDir = Paths.get("uploads-temp", safeService, safeName, safeIdentifier);
Files.createDirectories(tempDir);
java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index);
Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING);
return Response.ok("Chunk " + index + " received").build();
} catch (IOException e) {
LOGGER.error("Failed to write chunk {} for service='{}', name='{}', identifier='{}'", index, serviceString, name, identifier, e);
return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build();
}
}
@POST
@Path("/{service}/{name}/{identifier}/finalize")
@Produces(MediaType.TEXT_PLAIN)
@Operation(
summary = "Finalize a chunked upload and build a raw, unsigned, ARBITRARY transaction",
responses = {
@ApiResponse(
description = "raw, unsigned, ARBITRARY transaction encoded in Base58",
content = @Content(mediaType = MediaType.TEXT_PLAIN)
)
}
)
@SecurityRequirement(name = "apiKey")
public String finalizeUpload(
@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") String serviceString,
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@QueryParam("title") String title,
@QueryParam("description") String description,
@QueryParam("tags") List<String> tags,
@QueryParam("category") Category category,
@QueryParam("filename") String filename,
@QueryParam("fee") Long fee,
@QueryParam("preview") Boolean preview,
@QueryParam("isZip") Boolean isZip
) {
Security.checkApiCallAllowed(request);
java.nio.file.Path tempFile = null;
java.nio.file.Path tempDir = null;
java.nio.file.Path chunkDir = null;
try {
String safeService = Paths.get(serviceString).getFileName().toString();
String safeName = Paths.get(name).getFileName().toString();
String safeIdentifier = Paths.get(identifier).getFileName().toString();
java.nio.file.Path baseUploadsDir = Paths.get("uploads-temp"); // relative to Qortal working dir
chunkDir = baseUploadsDir.resolve(safeService).resolve(safeName).resolve(safeIdentifier);
if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload");
}
// Step 1: Determine a safe filename for disk temp file (regardless of extension correctness)
String safeFilename = filename;
if (filename == null || filename.isBlank()) {
safeFilename = "qortal-" + NTP.getTime();
}
tempDir = Files.createTempDirectory("qortal-");
String sanitizedFilename = Paths.get(safeFilename).getFileName().toString();
tempFile = tempDir.resolve(sanitizedFilename);
// Step 2: Merge chunks
try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
byte[] buffer = new byte[65536];
for (java.nio.file.Path chunk : Files.list(chunkDir)
.filter(path -> path.getFileName().toString().startsWith("chunk_"))
.sorted(Comparator.comparingInt(path -> {
String name2 = path.getFileName().toString();
String numberPart = name2.substring("chunk_".length());
return Integer.parseInt(numberPart);
})).collect(Collectors.toList())) {
try (InputStream in = Files.newInputStream(chunk)) {
int bytesRead;
while ((bytesRead = in.read(buffer)) != -1) {
out.write(buffer, 0, bytesRead);
}
}
}
}
// Step 3: Determine correct extension
String detectedExtension = "";
String uploadFilename = null;
boolean extensionIsValid = false;
if (filename != null && !filename.isBlank()) {
int lastDot = filename.lastIndexOf('.');
if (lastDot > 0 && lastDot < filename.length() - 1) {
extensionIsValid = true;
uploadFilename = filename;
}
}
if (!extensionIsValid) {
Tika tika = new Tika();
String mimeType = tika.detect(tempFile.toFile());
try {
MimeTypes allTypes = MimeTypes.getDefaultMimeTypes();
org.apache.tika.mime.MimeType mime = allTypes.forName(mimeType);
detectedExtension = mime.getExtension();
} catch (MimeTypeException e) {
LOGGER.warn("Could not determine file extension for MIME type: {}", mimeType, e);
}
if (filename != null && !filename.isBlank()) {
int lastDot = filename.lastIndexOf('.');
String baseName = (lastDot > 0) ? filename.substring(0, lastDot) : filename;
uploadFilename = baseName + (detectedExtension != null ? detectedExtension : "");
} else {
uploadFilename = "qortal-" + NTP.getTime() + (detectedExtension != null ? detectedExtension : "");
}
}
Boolean isZipBoolean = false;
if (isZip != null && isZip) {
isZipBoolean = true;
}
return this.upload(
Service.valueOf(serviceString),
name,
identifier,
tempFile.toString(),
null,
null,
isZipBoolean,
fee,
uploadFilename,
title,
description,
tags,
category,
preview
);
} catch (IOException e) {
LOGGER.error("Unexpected error in finalizeUpload for service='{}', name='{}', name='{}'", serviceString, name, identifier, e);
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage());
} finally {
if (tempDir != null) {
try {
Files.walk(tempDir)
.sorted(Comparator.reverseOrder())
.map(java.nio.file.Path::toFile)
.forEach(File::delete);
} catch (IOException e) {
LOGGER.warn("Failed to delete temp directory: {}", tempDir, e);
}
}
try {
Files.walk(chunkDir)
.sorted(Comparator.reverseOrder())
.map(java.nio.file.Path::toFile)
.forEach(File::delete);
} catch (IOException e) {
LOGGER.warn("Failed to delete chunk directory: {}", chunkDir, e);
}
}
}
// Upload base64-encoded data
@ -1343,7 +1818,7 @@ public class ArbitraryResource {
if (path == null) {
// See if we have a string instead
if (string != null) {
if (filename == null) {
if (filename == null || filename.isBlank()) {
// Use current time as filename
filename = String.format("qortal-%d", NTP.getTime());
}
@ -1358,7 +1833,7 @@ public class ArbitraryResource {
}
// ... or base64 encoded raw data
else if (base64 != null) {
if (filename == null) {
if (filename == null || filename.isBlank()) {
// Use current time as filename
filename = String.format("qortal-%d", NTP.getTime());
}
@ -1409,6 +1884,7 @@ public class ArbitraryResource {
);
transactionBuilder.build();
// Don't compute nonce - this is done by the client (or via POST /arbitrary/compute)
ArbitraryTransactionData transactionData = transactionBuilder.getArbitraryTransactionData();
return Base58.encode(ArbitraryTransactionTransformer.toBytes(transactionData));
@ -1424,9 +1900,9 @@ public class ArbitraryResource {
}
}
private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) {
private void download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts, boolean attachment, String attachmentFilename) {
try {
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
int attempts = 0;
@ -1438,8 +1914,7 @@ public class ArbitraryResource {
if (async) {
// Asynchronous
arbitraryDataReader.loadAsynchronously(false, 1);
}
else {
} else {
// Synchronous
while (!Controller.isStopping()) {
attempts++;
@ -1449,7 +1924,6 @@ public class ArbitraryResource {
break;
} catch (MissingDataException e) {
if (attempts > maxAttempts) {
// Give up after 5 attempts
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later.");
}
}
@ -1470,68 +1944,155 @@ public class ArbitraryResource {
if (files != null && files.length == 1) {
// This is a single file resource
filepath = files[0];
}
else {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA,
"filepath is required for resources containing more than one file");
} else {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "filepath is required for resources containing more than one file");
}
}
java.nio.file.Path path = Paths.get(outputPath.toString(), filepath);
if (!Files.exists(path)) {
String message = String.format("No file exists at filepath: %s", filepath);
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message);
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No file exists at filepath: " + filepath);
}
byte[] data;
int fileSize = (int)path.toFile().length();
int length = fileSize;
if (attachment) {
String rawFilename;
// Parse "Range" header
Integer rangeStart = null;
Integer rangeEnd = null;
String range = request.getHeader("Range");
if (range != null) {
range = range.replace("bytes=", "");
String[] parts = range.split("-");
rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null;
rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize;
if (attachmentFilename != null && !attachmentFilename.isEmpty()) {
// 1. Sanitize first
String safeAttachmentFilename = attachmentFilename.replaceAll("[\\\\/:*?\"<>|]", "_");
// 2. Check for a valid extension (3–5 alphanumeric chars)
if (!safeAttachmentFilename.matches(".*\\.[a-zA-Z0-9]{2,5}$")) {
safeAttachmentFilename += ".bin";
}
rawFilename = safeAttachmentFilename;
} else {
// Fallback if no filename is provided
String baseFilename = (identifier != null && !identifier.isEmpty())
? name + "-" + identifier
: name;
rawFilename = baseFilename.replaceAll("[\\\\/:*?\"<>|]", "_") + ".bin";
}
// Optional: trim length
rawFilename = rawFilename.length() > 100 ? rawFilename.substring(0, 100) : rawFilename;
// 3. Set Content-Disposition header
response.setHeader("Content-Disposition", "attachment; filename=\"" + rawFilename + "\"");
}
// Determine the total size of the requested file
long fileSize = Files.size(path);
String mimeType = context.getMimeType(path.toString());
if (rangeStart != null && rangeEnd != null) {
// We have a range, so update the requested length
length = rangeEnd - rangeStart;
// Attempt to read the "Range" header from the request to support partial content delivery (e.g., for video streaming or resumable downloads)
String range = request.getHeader("Range");
long rangeStart = 0;
long rangeEnd = fileSize - 1;
boolean isPartial = false;
// If a Range header is present and no base64 encoding is requested, parse the range values
if (range != null && encoding == null) {
range = range.replace("bytes=", ""); // Remove the "bytes=" prefix
String[] parts = range.split("-"); // Split the range into start and end
// Parse range start
if (parts.length > 0 && !parts[0].isEmpty()) {
rangeStart = Long.parseLong(parts[0]);
}
// Parse range end, if present
if (parts.length > 1 && !parts[1].isEmpty()) {
rangeEnd = Long.parseLong(parts[1]);
}
isPartial = true; // Indicate that this is a partial content request
}
// Calculate how many bytes should be sent in the response
long contentLength = rangeEnd - rangeStart + 1;
// Inform the client that byte ranges are supported
response.setHeader("Accept-Ranges", "bytes");
if (isPartial) {
// If partial content was requested, return 206 Partial Content with appropriate headers
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize));
} else {
// Otherwise, return the entire file with status 200 OK
response.setStatus(HttpServletResponse.SC_OK);
}
// Initialize output streams for writing the file to the response
OutputStream rawOut = response.getOutputStream();
OutputStream base64Out = null;
OutputStream gzipOut = null;
if (encoding != null && "base64".equalsIgnoreCase(encoding)) {
// If base64 encoding is requested, override content type
response.setContentType("text/plain");
// Check if the client accepts gzip encoding
String acceptEncoding = request.getHeader("Accept-Encoding");
boolean wantsGzip = acceptEncoding != null && acceptEncoding.contains("gzip");
if (wantsGzip) {
// Wrap output in GZIP and Base64 streams if gzip is accepted
response.setHeader("Content-Encoding", "gzip");
gzipOut = new GZIPOutputStream(rawOut);
base64Out = java.util.Base64.getEncoder().wrap(gzipOut);
} else {
// Wrap output in Base64 only
base64Out = java.util.Base64.getEncoder().wrap(rawOut);
}
rawOut = base64Out; // Use the wrapped stream for writing
} else {
// For raw binary output, set the content type and length
response.setContentType(mimeType != null ? mimeType : "application/octet-stream");
response.setContentLength((int) contentLength);
}
// Stream file content
try (InputStream inputStream = Files.newInputStream(path)) {
if (rangeStart > 0) {
inputStream.skip(rangeStart);
}
byte[] buffer = new byte[65536];
long bytesRemaining = contentLength;
int bytesRead;
while (bytesRemaining > 0 && (bytesRead = inputStream.read(buffer, 0, (int) Math.min(buffer.length, bytesRemaining))) != -1) {
rawOut.write(buffer, 0, bytesRead);
bytesRemaining -= bytesRead;
}
}
// Stream finished
if (base64Out != null) {
base64Out.close(); // Also flushes and closes the wrapped gzipOut
} else if (gzipOut != null) {
gzipOut.close(); // Only close gzipOut if it wasn't wrapped by base64Out
} else {
rawOut.flush(); // Flush only the base output stream if nothing was wrapped
}
if (!response.isCommitted()) {
response.setStatus(HttpServletResponse.SC_OK);
response.getWriter().write(" ");
}
if (length < fileSize && encoding == null) {
// Partial content requested, and not encoding the data
response.setStatus(206);
response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize));
data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length);
}
else {
// Full content requested (or encoded data)
response.setStatus(200);
data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory
}
// Encode the data if requested
if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) {
data = Base64.encode(data);
}
response.addHeader("Accept-Ranges", "bytes");
response.setContentType(context.getMimeType(path.toString()));
response.setContentLength(data.length);
response.getOutputStream().write(data);
return response;
} catch (Exception e) {
LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage()));
} catch (IOException | InterruptedException | ApiException | DataException e) {
LOGGER.error(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e);
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage());
}
catch ( NumberFormatException e) {
LOGGER.error(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e);
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage());
}
}
private FileProperties getFileProperties(Service service, String name, String identifier) {
try {
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);

2
src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java
View File

@ -52,7 +52,7 @@ public class ArbitraryDataFile {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class);
public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB
public static final long MAX_FILE_SIZE = 2L * 1024 * 1024 * 1024; // 2 GiB
protected static final int MAX_CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB
public static final int CHUNK_SIZE = 512 * 1024; // 0.5MiB
public static int SHORT_DIGEST_LENGTH = 8;

5
src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java
View File

@ -29,6 +29,7 @@ import org.qortal.utils.FilesystemUtils;
import org.qortal.utils.NTP;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
@ -197,7 +198,7 @@ public class ArbitraryDataTransactionBuilder {
// We can't use PATCH for on-chain data because this requires the .qortal directory, which can't be put on chain
final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(this.path, false);
final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE);
final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE);
if (shouldUseOnChainData) {
LOGGER.info("Data size is small enough to go on chain - using PUT");
return Method.PUT;
@ -245,7 +246,7 @@ public class ArbitraryDataTransactionBuilder {
// Single file resources are handled differently, especially for very small data payloads, as these go on chain
final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(path, false);
final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE);
final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE);
// Use zip compression if data isn't going on chain
Compression compression = shouldUseOnChainData ? Compression.NONE : Compression.ZIP;

4
src/main/java/org/qortal/crypto/AES.java
View File

@ -100,7 +100,7 @@ public class AES {
// Prepend the output stream with the 16 byte initialization vector
outputStream.write(iv.getIV());
byte[] buffer = new byte[1024];
byte[] buffer = new byte[65536];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
byte[] output = cipher.update(buffer, 0, bytesRead);
@ -138,7 +138,7 @@ public class AES {
Cipher cipher = Cipher.getInstance(algorithm);
cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv));
byte[] buffer = new byte[64];
byte[] buffer = new byte[65536];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
byte[] output = cipher.update(buffer, 0, bytesRead);

43
src/main/java/org/qortal/utils/FilesystemUtils.java
View File

@ -6,6 +6,7 @@ import org.qortal.settings.Settings;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
@ -232,32 +233,38 @@ public class FilesystemUtils {
}
public static byte[] getSingleFileContents(Path path, Integer maxLength) throws IOException {
byte[] data = null;
// TODO: limit the file size that can be loaded into memory
Path filePath = null;
// If the path is a file, read the contents directly
if (path.toFile().isFile()) {
int fileSize = (int)path.toFile().length();
maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize;
data = FilesystemUtils.readFromFile(path.toString(), 0, maxLength);
}
// Or if it's a directory, only load file contents if there is a single file inside it
else if (path.toFile().isDirectory()) {
if (Files.isRegularFile(path)) {
filePath = path;
} else if (Files.isDirectory(path)) {
String[] files = ArrayUtils.removeElement(path.toFile().list(), ".qortal");
if (files.length == 1) {
Path filePath = Paths.get(path.toString(), files[0]);
if (filePath.toFile().isFile()) {
int fileSize = (int)filePath.toFile().length();
maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize;
data = FilesystemUtils.readFromFile(filePath.toString(), 0, maxLength);
}
filePath = path.resolve(files[0]);
}
}
return data;
if (filePath == null || !Files.exists(filePath)) {
return null;
}
long fileSize = Files.size(filePath);
int length = (maxLength != null) ? Math.min(maxLength, (int) Math.min(fileSize, Integer.MAX_VALUE)) : (int) Math.min(fileSize, Integer.MAX_VALUE);
try (InputStream in = Files.newInputStream(filePath)) {
byte[] buffer = new byte[length];
int bytesRead = in.read(buffer);
if (bytesRead < length) {
// Resize buffer to actual read size
byte[] trimmed = new byte[bytesRead];
System.arraycopy(buffer, 0, trimmed, 0, bytesRead);
return trimmed;
}
return buffer;
}
}
/**
* isSingleFileResource
* Returns true if the path points to a file, or a

53
src/main/java/org/qortal/utils/ZipUtils.java
View File

@ -27,6 +27,8 @@
package org.qortal.utils;
import java.io.BufferedOutputStream;
import org.qortal.controller.Controller;
import java.io.File;
@ -44,12 +46,16 @@ public class ZipUtils {
File sourceFile = new File(sourcePath);
boolean isSingleFile = Paths.get(sourcePath).toFile().isFile();
FileOutputStream fileOutputStream = new FileOutputStream(destFilePath);
// 🔧 Use best speed compression level
ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream);
ZipUtils.zip(sourceFile, enclosingFolderName, zipOutputStream, isSingleFile);
zipOutputStream.close();
fileOutputStream.close();
}
public static void zip(final File fileToZip, final String enclosingFolderName, final ZipOutputStream zipOut, boolean isSingleFile) throws IOException, InterruptedException {
if (Controller.isStopping()) {
throw new InterruptedException("Controller is stopping");
@ -82,7 +88,7 @@ public class ZipUtils {
final FileInputStream fis = new FileInputStream(fileToZip);
final ZipEntry zipEntry = new ZipEntry(enclosingFolderName);
zipOut.putNextEntry(zipEntry);
final byte[] bytes = new byte[1024];
final byte[] bytes = new byte[65536];
int length;
while ((length = fis.read(bytes)) >= 0) {
zipOut.write(bytes, 0, length);
@ -92,34 +98,35 @@ public class ZipUtils {
public static void unzip(String sourcePath, String destPath) throws IOException {
final File destDir = new File(destPath);
final byte[] buffer = new byte[1024];
final ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath));
ZipEntry zipEntry = zis.getNextEntry();
while (zipEntry != null) {
final File newFile = ZipUtils.newFile(destDir, zipEntry);
if (zipEntry.isDirectory()) {
if (!newFile.isDirectory() && !newFile.mkdirs()) {
throw new IOException("Failed to create directory " + newFile);
}
} else {
File parent = newFile.getParentFile();
if (!parent.isDirectory() && !parent.mkdirs()) {
throw new IOException("Failed to create directory " + parent);
}
final byte[] buffer = new byte[65536];
try (ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath))) {
ZipEntry zipEntry = zis.getNextEntry();
while (zipEntry != null) {
final File newFile = ZipUtils.newFile(destDir, zipEntry);
if (zipEntry.isDirectory()) {
if (!newFile.isDirectory() && !newFile.mkdirs()) {
throw new IOException("Failed to create directory " + newFile);
}
} else {
File parent = newFile.getParentFile();
if (!parent.isDirectory() && !parent.mkdirs()) {
throw new IOException("Failed to create directory " + parent);
}
final FileOutputStream fos = new FileOutputStream(newFile);
int len;
while ((len = zis.read(buffer)) > 0) {
fos.write(buffer, 0, len);
try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(newFile), buffer.length)) {
int len;
while ((len = zis.read(buffer)) > 0) {
bos.write(buffer, 0, len);
}
}
}
fos.close();
zipEntry = zis.getNextEntry();
}
zipEntry = zis.getNextEntry();
zis.closeEntry();
}
zis.closeEntry();
zis.close();
}
/**
* See: https://snyk.io/research/zip-slip-vulnerability
*/