Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ object DatasetResource {
class DatasetResource {
private val ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE = "User has no access to this dataset"
private val ERR_DATASET_VERSION_NOT_FOUND_MESSAGE = "The version of the dataset not found"
private val EXPIRATION_MINUTES = 5

/**
* Helper function to get the dataset from DB with additional information including user access privilege and owner email
Expand Down Expand Up @@ -564,16 +565,37 @@ class DatasetResource {
generatePresignedResponse(encodedUrl, datasetName, commitHash, uid)
}

@GET
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("/presign-download-s3")
def getPresignedUrlWithS3(
@QueryParam("filePath") encodedUrl: String,
@QueryParam("datasetName") datasetName: String,
@QueryParam("commitHash") commitHash: String,
@Auth user: SessionUser
): Response = {
val uid = user.getUid
generatePresignedResponseWithS3(encodedUrl, datasetName, commitHash, uid)
}

@GET
@Path("/public-presign-download")
def getPublicPresignedUrl(
@QueryParam("filePath") encodedUrl: String,
@QueryParam("datasetName") datasetName: String,
@QueryParam("commitHash") commitHash: String
): Response = {
val user = new SessionUser(new User())
val uid = user.getUid
generatePresignedResponse(encodedUrl, datasetName, commitHash, uid)
generatePresignedResponse(encodedUrl, datasetName, commitHash, null)
}

@GET
@Path("/public-presign-download-s3")
def getPublicPresignedUrlWithS3(
@QueryParam("filePath") encodedUrl: String,
@QueryParam("datasetName") datasetName: String,
@QueryParam("commitHash") commitHash: String
): Response = {
generatePresignedResponseWithS3(encodedUrl, datasetName, commitHash, null)
}

@DELETE
Expand Down Expand Up @@ -1200,34 +1222,83 @@ class DatasetResource {
commitHash: String,
uid: Integer
): Response = {
resolveDatasetAndPath(encodedUrl, datasetName, commitHash, uid) match {
case Left(errorResponse) =>
errorResponse

case Right((resolvedDatasetName, resolvedCommitHash, resolvedFilePath)) =>
val url = LakeFSStorageClient.getFilePresignedUrl(
resolvedDatasetName,
resolvedCommitHash,
resolvedFilePath
)

Response.ok(Map("presignedUrl" -> url)).build()
}
}

private def generatePresignedResponseWithS3(
encodedUrl: String,
datasetName: String,
commitHash: String,
uid: Integer
): Response = {
resolveDatasetAndPath(encodedUrl, datasetName, commitHash, uid) match {
case Left(errorResponse) =>
errorResponse

case Right((resolvedDatasetName, resolvedCommitHash, resolvedFilePath)) =>
val fileName = resolvedFilePath.split("/").lastOption.getOrElse("download")
val contentType = "application/octet-stream"
val url = S3StorageClient.getFilePresignedUrl(
resolvedDatasetName,
resolvedCommitHash,
resolvedFilePath,
fileName,
contentType,
EXPIRATION_MINUTES
)

Response.ok(Map("presignedUrl" -> url)).build()
}
}

private def resolveDatasetAndPath(
encodedUrl: String,
datasetName: String,
commitHash: String,
uid: Integer
): Either[Response, (String, String, String)] = {
val decodedPathStr = URLDecoder.decode(encodedUrl, StandardCharsets.UTF_8.name())

(Option(datasetName), Option(commitHash)) match {
case (Some(_), None) | (None, Some(_)) =>
// Case 1: Only one parameter is provided (error case)
Response
.status(Response.Status.BAD_REQUEST)
.entity(
"Both datasetName and commitHash must be provided together, or neither should be provided."
)
.build()
Left(
Response
.status(Response.Status.BAD_REQUEST)
.entity(
"Both datasetName and commitHash must be provided together, or neither should be provided."
)
.build()
)

case (Some(dsName), Some(commit)) =>
// Case 2: datasetName and commitHash are provided, validate access
withTransaction(context) { ctx =>
val response = withTransaction(context) { ctx =>
val datasetDao = new DatasetDao(ctx.configuration())
val datasets = datasetDao.fetchByName(dsName).asScala.toList

if (datasets.isEmpty || !userHasReadAccess(ctx, datasets.head.getDid, uid))
throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE)

val url = LakeFSStorageClient.getFilePresignedUrl(dsName, commit, decodedPathStr)
Response.ok(Map("presignedUrl" -> url)).build()
(dsName, commit, decodedPathStr)
}
Right(response)

case (None, None) =>
// Case 3: Neither datasetName nor commitHash are provided, resolve normally
withTransaction(context) { ctx =>
val response = withTransaction(context) { ctx =>
val fileUri = FileResolver.resolve(decodedPathStr)
val document = DocumentFactory.openReadonlyDocument(fileUri).asInstanceOf[OnDataset]
val datasetDao = new DatasetDao(ctx.configuration())
Expand All @@ -1236,18 +1307,13 @@ class DatasetResource {
if (datasets.isEmpty || !userHasReadAccess(ctx, datasets.head.getDid, uid))
throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE)

Response
.ok(
Map(
"presignedUrl" -> LakeFSStorageClient.getFilePresignedUrl(
document.getDatasetName(),
document.getVersionHash(),
document.getFileRelativePath()
)
)
)
.build()
(
document.getDatasetName(),
document.getVersionHash(),
document.getFileRelativePath()
)
}
Right(response)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,12 @@ import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCrede
import software.amazon.awssdk.regions.Region
import software.amazon.awssdk.services.s3.{S3Client, S3Configuration}
import software.amazon.awssdk.services.s3.model._
import software.amazon.awssdk.services.s3.presigner.S3Presigner
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest
import software.amazon.awssdk.services.s3.model.GetObjectRequest

import java.net.URI
import java.time.Duration
import java.security.MessageDigest
import scala.jdk.CollectionConverters._

Expand All @@ -36,10 +41,10 @@ import scala.jdk.CollectionConverters._
object S3StorageClient {
val MINIMUM_NUM_OF_MULTIPART_S3_PART: Long = 5L * 1024 * 1024 // 5 MiB
val MAXIMUM_NUM_OF_MULTIPART_S3_PARTS = 10_000
val credentials = AwsBasicCredentials.create(StorageConfig.s3Username, StorageConfig.s3Password)

// Initialize MinIO-compatible S3 Client
private lazy val s3Client: S3Client = {
val credentials = AwsBasicCredentials.create(StorageConfig.s3Username, StorageConfig.s3Password)
S3Client
.builder()
.credentialsProvider(StaticCredentialsProvider.create(credentials))
Expand All @@ -51,6 +56,29 @@ object S3StorageClient {
.build()
}

// Initialize S3-compatible presigner for LakeFS S3 Gateway
private lazy val s3Presigner: S3Presigner = {
val fullUri = new URI(StorageConfig.lakefsEndpoint)
val baseUri = new URI(
fullUri.getScheme,
null,
fullUri.getHost,
fullUri.getPort,
null,
null,
null
) // Extract just the base (scheme + host + port)
S3Presigner
.builder()
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.region(Region.of(StorageConfig.s3Region))
.endpointOverride(baseUri) // LakeFS base URL ("http://localhost:8000" on local)
.serviceConfiguration(
S3Configuration.builder().pathStyleAccessEnabled(true).build()
)
.build()
}

/**
* Checks if a directory (prefix) exists within an S3 bucket.
*
Expand Down Expand Up @@ -139,4 +167,41 @@ object S3StorageClient {
s3Client.deleteObjects(deleteObjectsRequest)
}
}

/**
* Retrieves file content from a specific commit and path.
*
* @param repoName Repository name.
* @param commitHash Commit hash of the version.
* @param filePath Path to the file in the repository.
* @param fileName Name of the file downloaded via the presigned URL.
* @param contentType Type of the file downloaded via the presigned URL.
* @param expirationMinutes Duration in minutes that the presigned URL is valid.
*/
def getFilePresignedUrl(
repoName: String,
commitHash: String,
filePath: String,
fileName: String,
contentType: String,
expirationMinutes: Long
): String = {
val getObjectRequest = GetObjectRequest
.builder()
.bucket(repoName)
.key(s"$commitHash/$filePath")
.responseContentDisposition(s"attachment; filename='$fileName'")
.responseContentType(contentType)
.build()

val presignRequest = GetObjectPresignRequest
.builder()
.signatureDuration(Duration.ofMinutes(expirationMinutes))
.getObjectRequest(getObjectRequest)
.build()

val presignedUrl = s3Presigner.presignGetObject(presignRequest).url().toString
s3Presigner.close()
presignedUrl
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -269,8 +269,7 @@ export class DatasetDetailComponent implements OnInit {

onClickDownloadCurrentFile = (): void => {
if (!this.did || !this.selectedVersion?.dvid) return;

this.downloadService.downloadSingleFile(this.currentDisplayedFileName).pipe(untilDestroyed(this)).subscribe();
this.downloadService.downloadSingleFile(this.currentDisplayedFileName);
};

onClickScaleTheView() {
Expand Down
23 changes: 23 additions & 0 deletions core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,29 @@ export class DatasetService {
.pipe(switchMap(({ presignedUrl }) => this.http.get(presignedUrl, { responseType: "blob" })));
}

/**
* Retrieves a single file from a dataset version using a pre-signed URL.
* @param filePath Relative file path within the dataset.
* @param isLogin Determine whether a user is currently logged in
* @returns void File is downloaded natively by the browser.
*/
public retrieveDatasetVersionSingleFileViaBrowser(filePath: string, isLogin: boolean = true): void {
const endpointSegment = isLogin ? "presign-download-s3" : "public-presign-download-s3";
const endpoint = `${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}/${endpointSegment}?filePath=${encodeURIComponent(filePath)}`;

this.http.get<{ presignedUrl: string }>(endpoint).subscribe({
next: response => {
const presignedUrl = response.presignedUrl;
const downloadUrl = document.createElement("a");

downloadUrl.href = presignedUrl;
document.body.appendChild(downloadUrl);
downloadUrl.click();
downloadUrl.remove();
},
});
}

/**
* Retrieves a zip file of a dataset version.
* @param did Dataset ID
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,48 +60,6 @@ describe("DownloadService", () => {
notificationServiceSpy = TestBed.inject(NotificationService) as jasmine.SpyObj<NotificationService>;
});

it("should download a single file successfully", (done: DoneFn) => {
const filePath = "test/file.txt";
const mockBlob = new Blob(["test content"], { type: "text/plain" });

datasetServiceSpy.retrieveDatasetVersionSingleFile.and.returnValue(of(mockBlob));

downloadService.downloadSingleFile(filePath).subscribe({
next: blob => {
expect(blob).toBe(mockBlob);
expect(notificationServiceSpy.info).toHaveBeenCalledWith("Starting to download file test/file.txt");
expect(datasetServiceSpy.retrieveDatasetVersionSingleFile).toHaveBeenCalledWith(filePath);
expect(fileSaverServiceSpy.saveAs).toHaveBeenCalledWith(mockBlob, "file.txt");
expect(notificationServiceSpy.success).toHaveBeenCalledWith("File test/file.txt has been downloaded");
done();
},
error: (error: unknown) => {
fail("Should not have thrown an error: " + error);
},
});
});

it("should handle download failure correctly", done => {
const filePath = "test/file.txt";
const errorMessage = "Download failed";

datasetServiceSpy.retrieveDatasetVersionSingleFile.and.returnValue(throwError(() => new Error(errorMessage)));

downloadService.downloadSingleFile(filePath).subscribe({
next: () => {
fail("Should have thrown an error");
},
error: (error: unknown) => {
expect(error).toBeTruthy();
expect(notificationServiceSpy.info).toHaveBeenCalledWith("Starting to download file test/file.txt");
expect(datasetServiceSpy.retrieveDatasetVersionSingleFile).toHaveBeenCalledWith(filePath);
expect(fileSaverServiceSpy.saveAs).not.toHaveBeenCalled();
expect(notificationServiceSpy.error).toHaveBeenCalledWith("Error downloading file 'test/file.txt'");
done();
},
});
});

it("should download a dataset successfully", done => {
const datasetId = 1;
const datasetName = "TestDataset";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,16 +93,9 @@ export class DownloadService {
);
}

downloadSingleFile(filePath: string): Observable<Blob> {
const DEFAULT_FILE_NAME = "download";
const fileName = filePath.split("/").pop() || DEFAULT_FILE_NAME;
return this.downloadWithNotification(
() => this.datasetService.retrieveDatasetVersionSingleFile(filePath),
fileName,
`Starting to download file ${filePath}`,
`File ${filePath} has been downloaded`,
`Error downloading file '${filePath}'`
);
downloadSingleFile(filePath: string): void {
this.notificationService.info(`Starting to download file ${filePath}`);
this.datasetService.retrieveDatasetVersionSingleFileViaBrowser(filePath);
}

downloadWorkflowsAsZip(workflowEntries: Array<{ id: number; name: string }>): Observable<Blob> {
Expand Down
Loading