diff --git a/core/file-service/src/main/scala/edu/uci/ics/texera/service/resource/DatasetResource.scala b/core/file-service/src/main/scala/edu/uci/ics/texera/service/resource/DatasetResource.scala index 37e48f66318..55722395aa8 100644 --- a/core/file-service/src/main/scala/edu/uci/ics/texera/service/resource/DatasetResource.scala +++ b/core/file-service/src/main/scala/edu/uci/ics/texera/service/resource/DatasetResource.scala @@ -195,6 +195,7 @@ object DatasetResource { class DatasetResource { private val ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE = "User has no access to this dataset" private val ERR_DATASET_VERSION_NOT_FOUND_MESSAGE = "The version of the dataset not found" + private val EXPIRATION_MINUTES = 5 /** * Helper function to get the dataset from DB with additional information including user access privilege and owner email @@ -564,6 +565,19 @@ class DatasetResource { generatePresignedResponse(encodedUrl, datasetName, commitHash, uid) } + @GET + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/presign-download-s3") + def getPresignedUrlWithS3( + @QueryParam("filePath") encodedUrl: String, + @QueryParam("datasetName") datasetName: String, + @QueryParam("commitHash") commitHash: String, + @Auth user: SessionUser + ): Response = { + val uid = user.getUid + generatePresignedResponseWithS3(encodedUrl, datasetName, commitHash, uid) + } + @GET @Path("/public-presign-download") def getPublicPresignedUrl( @@ -571,9 +585,17 @@ class DatasetResource { @QueryParam("datasetName") datasetName: String, @QueryParam("commitHash") commitHash: String ): Response = { - val user = new SessionUser(new User()) - val uid = user.getUid - generatePresignedResponse(encodedUrl, datasetName, commitHash, uid) + generatePresignedResponse(encodedUrl, datasetName, commitHash, null) + } + + @GET + @Path("/public-presign-download-s3") + def getPublicPresignedUrlWithS3( + @QueryParam("filePath") encodedUrl: String, + @QueryParam("datasetName") datasetName: String, + @QueryParam("commitHash") commitHash: String + ): Response = { + generatePresignedResponseWithS3(encodedUrl, datasetName, commitHash, null) } @DELETE @@ -1200,34 +1222,83 @@ class DatasetResource { commitHash: String, uid: Integer ): Response = { + resolveDatasetAndPath(encodedUrl, datasetName, commitHash, uid) match { + case Left(errorResponse) => + errorResponse + + case Right((resolvedDatasetName, resolvedCommitHash, resolvedFilePath)) => + val url = LakeFSStorageClient.getFilePresignedUrl( + resolvedDatasetName, + resolvedCommitHash, + resolvedFilePath + ) + + Response.ok(Map("presignedUrl" -> url)).build() + } + } + + private def generatePresignedResponseWithS3( + encodedUrl: String, + datasetName: String, + commitHash: String, + uid: Integer + ): Response = { + resolveDatasetAndPath(encodedUrl, datasetName, commitHash, uid) match { + case Left(errorResponse) => + errorResponse + + case Right((resolvedDatasetName, resolvedCommitHash, resolvedFilePath)) => + val fileName = resolvedFilePath.split("/").lastOption.getOrElse("download") + val contentType = "application/octet-stream" + val url = S3StorageClient.getFilePresignedUrl( + resolvedDatasetName, + resolvedCommitHash, + resolvedFilePath, + fileName, + contentType, + EXPIRATION_MINUTES + ) + + Response.ok(Map("presignedUrl" -> url)).build() + } + } + + private def resolveDatasetAndPath( + encodedUrl: String, + datasetName: String, + commitHash: String, + uid: Integer + ): Either[Response, (String, String, String)] = { val decodedPathStr = URLDecoder.decode(encodedUrl, StandardCharsets.UTF_8.name()) (Option(datasetName), Option(commitHash)) match { case (Some(_), None) | (None, Some(_)) => // Case 1: Only one parameter is provided (error case) - Response - .status(Response.Status.BAD_REQUEST) - .entity( - "Both datasetName and commitHash must be provided together, or neither should be provided." - ) - .build() + Left( + Response + .status(Response.Status.BAD_REQUEST) + .entity( + "Both datasetName and commitHash must be provided together, or neither should be provided." + ) + .build() + ) case (Some(dsName), Some(commit)) => // Case 2: datasetName and commitHash are provided, validate access - withTransaction(context) { ctx => + val response = withTransaction(context) { ctx => val datasetDao = new DatasetDao(ctx.configuration()) val datasets = datasetDao.fetchByName(dsName).asScala.toList if (datasets.isEmpty || !userHasReadAccess(ctx, datasets.head.getDid, uid)) throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE) - val url = LakeFSStorageClient.getFilePresignedUrl(dsName, commit, decodedPathStr) - Response.ok(Map("presignedUrl" -> url)).build() + (dsName, commit, decodedPathStr) } + Right(response) case (None, None) => // Case 3: Neither datasetName nor commitHash are provided, resolve normally - withTransaction(context) { ctx => + val response = withTransaction(context) { ctx => val fileUri = FileResolver.resolve(decodedPathStr) val document = DocumentFactory.openReadonlyDocument(fileUri).asInstanceOf[OnDataset] val datasetDao = new DatasetDao(ctx.configuration()) @@ -1236,18 +1307,13 @@ class DatasetResource { if (datasets.isEmpty || !userHasReadAccess(ctx, datasets.head.getDid, uid)) throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE) - Response - .ok( - Map( - "presignedUrl" -> LakeFSStorageClient.getFilePresignedUrl( - document.getDatasetName(), - document.getVersionHash(), - document.getFileRelativePath() - ) - ) - ) - .build() + ( + document.getDatasetName(), + document.getVersionHash(), + document.getFileRelativePath() + ) } + Right(response) } } } diff --git a/core/file-service/src/main/scala/edu/uci/ics/texera/service/util/S3StorageClient.scala b/core/file-service/src/main/scala/edu/uci/ics/texera/service/util/S3StorageClient.scala index 2b1afd1165b..ae110fec525 100644 --- a/core/file-service/src/main/scala/edu/uci/ics/texera/service/util/S3StorageClient.scala +++ b/core/file-service/src/main/scala/edu/uci/ics/texera/service/util/S3StorageClient.scala @@ -24,7 +24,12 @@ import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCrede import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.{S3Client, S3Configuration} import software.amazon.awssdk.services.s3.model._ +import software.amazon.awssdk.services.s3.presigner.S3Presigner +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest +import software.amazon.awssdk.services.s3.model.GetObjectRequest +import java.net.URI +import java.time.Duration import java.security.MessageDigest import scala.jdk.CollectionConverters._ @@ -36,10 +41,10 @@ import scala.jdk.CollectionConverters._ object S3StorageClient { val MINIMUM_NUM_OF_MULTIPART_S3_PART: Long = 5L * 1024 * 1024 // 5 MiB val MAXIMUM_NUM_OF_MULTIPART_S3_PARTS = 10_000 + val credentials = AwsBasicCredentials.create(StorageConfig.s3Username, StorageConfig.s3Password) // Initialize MinIO-compatible S3 Client private lazy val s3Client: S3Client = { - val credentials = AwsBasicCredentials.create(StorageConfig.s3Username, StorageConfig.s3Password) S3Client .builder() .credentialsProvider(StaticCredentialsProvider.create(credentials)) @@ -51,6 +56,29 @@ object S3StorageClient { .build() } + // Initialize S3-compatible presigner for LakeFS S3 Gateway + private lazy val s3Presigner: S3Presigner = { + val fullUri = new URI(StorageConfig.lakefsEndpoint) + val baseUri = new URI( + fullUri.getScheme, + null, + fullUri.getHost, + fullUri.getPort, + null, + null, + null + ) // Extract just the base (scheme + host + port) + S3Presigner + .builder() + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .region(Region.of(StorageConfig.s3Region)) + .endpointOverride(baseUri) // LakeFS base URL ("http://localhost:8000" on local) + .serviceConfiguration( + S3Configuration.builder().pathStyleAccessEnabled(true).build() + ) + .build() + } + /** * Checks if a directory (prefix) exists within an S3 bucket. * @@ -139,4 +167,41 @@ object S3StorageClient { s3Client.deleteObjects(deleteObjectsRequest) } } + + /** + * Retrieves file content from a specific commit and path. + * + * @param repoName Repository name. + * @param commitHash Commit hash of the version. + * @param filePath Path to the file in the repository. + * @param fileName Name of the file downloaded via the presigned URL. + * @param contentType Type of the file downloaded via the presigned URL. + * @param expirationMinutes Duration in minutes that the presigned URL is valid. + */ + def getFilePresignedUrl( + repoName: String, + commitHash: String, + filePath: String, + fileName: String, + contentType: String, + expirationMinutes: Long + ): String = { + val getObjectRequest = GetObjectRequest + .builder() + .bucket(repoName) + .key(s"$commitHash/$filePath") + .responseContentDisposition(s"attachment; filename='$fileName'") + .responseContentType(contentType) + .build() + + val presignRequest = GetObjectPresignRequest + .builder() + .signatureDuration(Duration.ofMinutes(expirationMinutes)) + .getObjectRequest(getObjectRequest) + .build() + + val presignedUrl = s3Presigner.presignGetObject(presignRequest).url().toString + s3Presigner.close() + presignedUrl + } } diff --git a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts index 9dd4de0c408..fb40c946bc0 100644 --- a/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts +++ b/core/gui/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts @@ -269,8 +269,7 @@ export class DatasetDetailComponent implements OnInit { onClickDownloadCurrentFile = (): void => { if (!this.did || !this.selectedVersion?.dvid) return; - - this.downloadService.downloadSingleFile(this.currentDisplayedFileName).pipe(untilDestroyed(this)).subscribe(); + this.downloadService.downloadSingleFile(this.currentDisplayedFileName); }; onClickScaleTheView() { diff --git a/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts b/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts index 226b1edb978..333be35fa37 100644 --- a/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts +++ b/core/gui/src/app/dashboard/service/user/dataset/dataset.service.ts @@ -93,6 +93,29 @@ export class DatasetService { .pipe(switchMap(({ presignedUrl }) => this.http.get(presignedUrl, { responseType: "blob" }))); } + /** + * Retrieves a single file from a dataset version using a pre-signed URL. + * @param filePath Relative file path within the dataset. + * @param isLogin Determine whether a user is currently logged in + * @returns void File is downloaded natively by the browser. + */ + public retrieveDatasetVersionSingleFileViaBrowser(filePath: string, isLogin: boolean = true): void { + const endpointSegment = isLogin ? "presign-download-s3" : "public-presign-download-s3"; + const endpoint = `${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}/${endpointSegment}?filePath=${encodeURIComponent(filePath)}`; + + this.http.get<{ presignedUrl: string }>(endpoint).subscribe({ + next: response => { + const presignedUrl = response.presignedUrl; + const downloadUrl = document.createElement("a"); + + downloadUrl.href = presignedUrl; + document.body.appendChild(downloadUrl); + downloadUrl.click(); + downloadUrl.remove(); + }, + }); + } + /** * Retrieves a zip file of a dataset version. * @param did Dataset ID diff --git a/core/gui/src/app/dashboard/service/user/download/download.service.spec.ts b/core/gui/src/app/dashboard/service/user/download/download.service.spec.ts index f6d5a4b4b1c..d1c55a434f2 100644 --- a/core/gui/src/app/dashboard/service/user/download/download.service.spec.ts +++ b/core/gui/src/app/dashboard/service/user/download/download.service.spec.ts @@ -60,48 +60,6 @@ describe("DownloadService", () => { notificationServiceSpy = TestBed.inject(NotificationService) as jasmine.SpyObj; }); - it("should download a single file successfully", (done: DoneFn) => { - const filePath = "test/file.txt"; - const mockBlob = new Blob(["test content"], { type: "text/plain" }); - - datasetServiceSpy.retrieveDatasetVersionSingleFile.and.returnValue(of(mockBlob)); - - downloadService.downloadSingleFile(filePath).subscribe({ - next: blob => { - expect(blob).toBe(mockBlob); - expect(notificationServiceSpy.info).toHaveBeenCalledWith("Starting to download file test/file.txt"); - expect(datasetServiceSpy.retrieveDatasetVersionSingleFile).toHaveBeenCalledWith(filePath); - expect(fileSaverServiceSpy.saveAs).toHaveBeenCalledWith(mockBlob, "file.txt"); - expect(notificationServiceSpy.success).toHaveBeenCalledWith("File test/file.txt has been downloaded"); - done(); - }, - error: (error: unknown) => { - fail("Should not have thrown an error: " + error); - }, - }); - }); - - it("should handle download failure correctly", done => { - const filePath = "test/file.txt"; - const errorMessage = "Download failed"; - - datasetServiceSpy.retrieveDatasetVersionSingleFile.and.returnValue(throwError(() => new Error(errorMessage))); - - downloadService.downloadSingleFile(filePath).subscribe({ - next: () => { - fail("Should have thrown an error"); - }, - error: (error: unknown) => { - expect(error).toBeTruthy(); - expect(notificationServiceSpy.info).toHaveBeenCalledWith("Starting to download file test/file.txt"); - expect(datasetServiceSpy.retrieveDatasetVersionSingleFile).toHaveBeenCalledWith(filePath); - expect(fileSaverServiceSpy.saveAs).not.toHaveBeenCalled(); - expect(notificationServiceSpy.error).toHaveBeenCalledWith("Error downloading file 'test/file.txt'"); - done(); - }, - }); - }); - it("should download a dataset successfully", done => { const datasetId = 1; const datasetName = "TestDataset"; diff --git a/core/gui/src/app/dashboard/service/user/download/download.service.ts b/core/gui/src/app/dashboard/service/user/download/download.service.ts index 813dea08e20..39b89f0f014 100644 --- a/core/gui/src/app/dashboard/service/user/download/download.service.ts +++ b/core/gui/src/app/dashboard/service/user/download/download.service.ts @@ -93,16 +93,9 @@ export class DownloadService { ); } - downloadSingleFile(filePath: string): Observable { - const DEFAULT_FILE_NAME = "download"; - const fileName = filePath.split("/").pop() || DEFAULT_FILE_NAME; - return this.downloadWithNotification( - () => this.datasetService.retrieveDatasetVersionSingleFile(filePath), - fileName, - `Starting to download file ${filePath}`, - `File ${filePath} has been downloaded`, - `Error downloading file '${filePath}'` - ); + downloadSingleFile(filePath: string): void { + this.notificationService.info(`Starting to download file ${filePath}`); + this.datasetService.retrieveDatasetVersionSingleFileViaBrowser(filePath); } downloadWorkflowsAsZip(workflowEntries: Array<{ id: number; name: string }>): Observable {