Skip to content

Check if file exists in object store before reusing #1801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
IvanBorislavovDimitrov wants to merge 2 commits into
base: master
Choose a base branch
from
+442 −134
Open

Check if file exists in object store before reusing #1801

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
19a1c3b
Check if file exists in object store before reusing
IvanBorislavovDimitrov Mar 25, 2026
6084326
Fix comments
IvanBorislavovDimitrov Mar 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions multiapps-controller-persistence/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,15 @@

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<java.util.logging.config.file>${project.basedir}/src/test/resources/logging.properties</java.util.logging.config.file>
</systemPropertyVariables>
</configuration>
</plugin>
<plugin>
<groupId>de.empulse.eclipselink</groupId>
<artifactId>staticweave-maven-plugin</artifactId>
Expand Down
2 changes: 2 additions & 0 deletions multiapps-controller-persistence/src/main/java/module-info.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
requires google.cloud.nio;
requires google.cloud.storage;
requires jakarta.xml.bind;
requires jakarta.annotation;
requires jakarta.inject;
requires liquibase.core;
requires org.apache.logging.log4j;
Expand All @@ -59,6 +60,7 @@
requires org.cloudfoundry.multiapps.common;
requires org.eclipse.persistence.core;
requires org.slf4j;
requires spring.beans;
requires spring.context;
requires spring.core;

Expand Down
7 changes: 0 additions & 7 deletions ...persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,11 @@ public final class Messages {
// Exception messages:
public static final String FILE_UPLOAD_FAILED = "Upload of file \"{0}\" to \"{1}\" failed";
public static final String FILE_NOT_FOUND = "File \"{0}\" not found";
public static final String FAILED_TO_UPDATE_SQL_QUERY = "Failed to update SQL query";
public static final String ERROR_FINDING_FILE_TO_UPLOAD = "Error finding file to upload with name {0}: {1}";
public static final String ERROR_READING_FILE_CONTENT = "Error reading content of file {0}: {1}";
public static final String FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST = "File with ID \"{0}\" and space \"{1}\" does not exist.";
public static final String ERROR_GETTING_FILES_WITH_SPACE_AND_NAMESPACE = "Error getting files with space {0} and namespace {1}";
public static final String ERROR_GETTING_FILES_WITH_SPACE_AND_OPERATION_ID = "Error getting files with space {0} and operation id {1}";
public static final String ERROR_GETTING_LOGS_WITH_SPACE_AND_OPERATION_ID = "Error getting logs with space {0} and operation id {1}";
public static final String ERROR_GETTING_FILES_WITH_SPACE_OPERATION_ID_AND_NAME = "Error getting files with space {0} operation id {1} and file name {2}";
public static final String ERROR_GETTING_LOGS_WITH_SPACE_OPERATION_ID_AND_NAME = "Error getting logs with space {0} operation id {1} and file name {2}";
public static final String ERROR_GETTING_ALL_FILES = "Error getting all files";
public static final String ERROR_LOG_FILE_NOT_FOUND = "Log file with name \"{0}\" for operation \"{1}\" in space \"{2}\" was not found";
Expand Down Expand Up @@ -61,16 +58,12 @@ public final class Messages {
public static final String COULD_NOT_CLOSE_RESULT_SET = "Could not close result set.";
public static final String COULD_NOT_CLOSE_STATEMENT = "Could not close statement.";
public static final String COULD_NOT_CLOSE_CONNECTION = "Could not close connection.";
public static final String COULD_NOT_CLOSE_LOGGER_CONTEXT = "Could not close logger context";
public static final String COULD_NOT_ROLLBACK_TRANSACTION = "Could not rollback transaction!";
public static final String COULD_NOT_PERSIST_LOGS_FILE = "Could not persist logs file: {0}";
public static final String ATTEMPT_TO_UPLOAD_BLOB_FAILED = "Attempt [{0}/{1}] to upload blob to ObjectStore failed with \"{2}\"";
public static final String ATTEMPT_TO_DOWNLOAD_MISSING_BLOB = "Attempt [{0}/{1}] to download missing blob {2} from ObjectStore";
public static final String USER_METADATA_OF_BLOB_0_EMPTY_AND_WILL_BE_DELETED = "User metadata of blob \"{0}\" is empty and will be deleted";
public static final String DATE_METADATA_OF_BLOB_0_IS_NOT_IN_PROPER_FORMAT_AND_WILL_BE_DELETED = "Date metadata of blob \"{0}\" is not in a proper format and will be deleted";

// INFO log messages:
public static final String DEFAULT_CONSOLE = "DefaultConsole";
public static final String DELETING_FILES_WITHOUT_CONTENT_WITH_IDS_0 = "Deleting files without content with ids: {0}";

// DEBUG log messages:
Expand Down
29 changes: 17 additions & 12 deletions ...g/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,5 @@
package org.cloudfoundry.multiapps.controller.persistence.services;

import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;

import com.azure.core.http.HttpClient;
import com.azure.core.http.okhttp.OkHttpAsyncHttpClientBuilder;
import com.azure.core.http.policy.ExponentialBackoffOptions;
Expand All @@ -30,8 +19,18 @@
import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants;
import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter;
import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;

public class AzureObjectStoreFileStorage implements FileStorage {
public class AzureObjectStoreFileStorage extends ObjectStoreFileStorage {

private static final String SAS_TOKEN = "sas_token";
private static final String CONTAINER_NAME = "container_name";
Expand Down Expand Up @@ -66,6 +65,12 @@ public List<FileEntry> getFileEntriesWithoutContent(List<FileEntry> fileEntries)
.toList();
}

@Override
protected boolean existsInObjectStore(FileEntry fileEntry) {
return containerClient.getBlobClient(fileEntry.getId())
.exists();
}

@Override
public void deleteFile(String id, String space) throws FileStorageException {
BlobClient blobClient = containerClient.getBlobClient(id);
Expand Down
34 changes: 24 additions & 10 deletions .../java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
package org.cloudfoundry.multiapps.controller.persistence.services;

import java.io.InputStream;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.util.List;

import org.cloudfoundry.multiapps.controller.persistence.Constants;
import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect;
import org.cloudfoundry.multiapps.controller.persistence.Messages;
import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry;
import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry;
import org.cloudfoundry.multiapps.controller.persistence.query.options.StreamFetchingOptions;
import org.cloudfoundry.multiapps.controller.persistence.query.providers.BlobSqlFileQueryProvider;
import org.cloudfoundry.multiapps.controller.persistence.query.providers.SqlFileQueryProvider;

import java.io.InputStream;
import java.sql.SQLException;
import java.text.MessageFormat;
import java.time.LocalDateTime;
import java.util.List;

public class DatabaseFileService extends FileService {

public DatabaseFileService(DataSourceWithDialect dataSourceWithDialect) {
Expand All @@ -27,15 +29,27 @@ protected DatabaseFileService(DataSourceWithDialect dataSourceWithDialect, SqlFi
super(dataSourceWithDialect, sqlFileQueryProvider, null);
}

@Override
public List<FileEntry> listFiles(String space, String namespace) throws FileStorageException {
try {
return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getListFilesQuery(space, namespace));
} catch (SQLException e) {
throw new FileStorageException(MessageFormat.format(Messages.ERROR_GETTING_FILES_WITH_SPACE_AND_NAMESPACE, space, namespace),
e);
}
}

@Override
public <T> T processFileContentWithOffset(FileContentToProcess fileContentToProcess, FileContentProcessor<T> fileContentProcessor)
throws FileStorageException {
try {
return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getProcessFileWithContentQueryWithOffsetQuery(fileContentToProcess.getSpaceGuid(),
fileContentToProcess.getGuid(),
new StreamFetchingOptions(fileContentToProcess.getStartOffset(),
fileContentToProcess.getEndOffset()),
fileContentProcessor));
return getSqlQueryExecutor().execute(
getSqlFileQueryProvider().getProcessFileWithContentQueryWithOffsetQuery(fileContentToProcess.getSpaceGuid(),
fileContentToProcess.getGuid(),
new StreamFetchingOptions(
fileContentToProcess.getStartOffset(),
fileContentToProcess.getEndOffset()),
fileContentProcessor));
} catch (SQLException e) {
throw new FileStorageException(e.getMessage(), e);
}
Expand Down
39 changes: 15 additions & 24 deletions ...src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,17 @@
package org.cloudfoundry.multiapps.controller.persistence.services;

import jakarta.xml.bind.DatatypeConverter;
import org.cloudfoundry.multiapps.controller.persistence.Constants;
import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect;
import org.cloudfoundry.multiapps.controller.persistence.Messages;
import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry;
import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry;
import org.cloudfoundry.multiapps.controller.persistence.query.providers.ExternalSqlFileQueryProvider;
import org.cloudfoundry.multiapps.controller.persistence.query.providers.SqlFileQueryProvider;
import org.cloudfoundry.multiapps.controller.persistence.util.SqlQueryExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
Expand All @@ -16,19 +28,6 @@
import java.util.List;
import java.util.UUID;

import org.cloudfoundry.multiapps.controller.persistence.Constants;
import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect;
import org.cloudfoundry.multiapps.controller.persistence.Messages;
import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry;
import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry;
import org.cloudfoundry.multiapps.controller.persistence.query.providers.ExternalSqlFileQueryProvider;
import org.cloudfoundry.multiapps.controller.persistence.query.providers.SqlFileQueryProvider;
import org.cloudfoundry.multiapps.controller.persistence.util.SqlQueryExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import jakarta.xml.bind.DatatypeConverter;

public class FileService {

protected static final String DEFAULT_TABLE_NAME = "LM_SL_PERSISTENCE_FILE";
Expand Down Expand Up @@ -77,23 +76,15 @@ public FileEntry addFile(FileEntry fileEntry, File existingFile) throws FileStor

public List<FileEntry> listFiles(String space, String namespace) throws FileStorageException {
try {
return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getListFilesQuery(space, namespace));
List<FileEntry> fileEntriesFromDb = getSqlQueryExecutor().execute(
getSqlFileQueryProvider().getListFilesQuery(space, namespace));
return fileStorage.getExistingFileEntries(fileEntriesFromDb);
} catch (SQLException e) {
throw new FileStorageException(MessageFormat.format(Messages.ERROR_GETTING_FILES_WITH_SPACE_AND_NAMESPACE, space, namespace),
e);
}
}

public List<FileEntry> listFilesBySpaceAndOperationId(String space, String operationId) throws FileStorageException {
try {
return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getListFilesBySpaceAndOperationId(space, operationId));
} catch (SQLException e) {
throw new FileStorageException(MessageFormat.format(Messages.ERROR_GETTING_FILES_WITH_SPACE_AND_OPERATION_ID, space,
operationId),
e);
}
}

public List<FileEntry> listFilesCreatedAfterAndBeforeWithoutOperationId(LocalDateTime after, LocalDateTime before)
throws FileStorageException {
try {
Expand Down
2 changes: 2 additions & 0 deletions ...src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ public interface FileStorage {
@Deprecated // This method is not reliable for aws as BlobStore::list might not return a complete list
List<FileEntry> getFileEntriesWithoutContent(List<FileEntry> fileEntries) throws FileStorageException;

List<FileEntry> getExistingFileEntries(List<FileEntry> fileEntries) throws FileStorageException;

void deleteFile(String id, String space) throws FileStorageException;

void deleteFilesBySpaceIds(List<String> spaceIds) throws FileStorageException;
Expand Down
47 changes: 33 additions & 14 deletions ...org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,5 @@
package org.cloudfoundry.multiapps.controller.persistence.services;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.channels.Channels;
import java.text.MessageFormat;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;

import com.google.api.gax.retrying.RetrySettings;
import com.google.auth.Credentials;
import com.google.auth.oauth2.GoogleCredentials;
Expand All @@ -32,6 +18,21 @@
import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper;
import org.springframework.http.MediaType;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.channels.Channels;
import java.text.MessageFormat;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;

public class GcpObjectStoreFileStorage implements FileStorage {

private final String bucketName;
Expand Down Expand Up @@ -104,6 +105,24 @@ public List<FileEntry> getFileEntriesWithoutContent(List<FileEntry> fileEntries)
.toList();
}

@Override
public List<FileEntry> getExistingFileEntries(List<FileEntry> fileEntries) {
if (fileEntries.isEmpty()) {
return List.of();
}
List<BlobId> blobIds = fileEntries.stream()
.map(fileEntry -> BlobId.of(bucketName, fileEntry.getId()))
.toList();
List<Blob> blobs = storage.get(blobIds);
Copy link
Contributor

@Yavor16 Yavor16 Mar 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this get request not executed in a thread like the other sdk-s?

Copy link
Contributor Author

@IvanBorislavovDimitrov IvanBorislavovDimitrov Mar 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

because it support multi fetch by multiple ids, others do not

Set<String> existingBlobNames = blobs.stream()
.filter(Objects::nonNull)
.map(Blob::getName)
.collect(Collectors.toSet());
return fileEntries.stream()
.filter(fileEntry -> existingBlobNames.contains(fileEntry.getId()))
.toList();
}

@Override
public void deleteFile(String id, String space) {
deleteFileWithGeneration(id);
Expand Down
29 changes: 17 additions & 12 deletions ...cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,5 @@
package org.cloudfoundry.multiapps.controller.persistence.services;

import java.io.IOException;
import java.io.InputStream;
import java.text.MessageFormat;
import java.time.LocalDateTime;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;

import org.cloudfoundry.multiapps.common.util.MiscUtil;
import org.cloudfoundry.multiapps.controller.persistence.Messages;
import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry;
Expand All @@ -30,7 +19,18 @@
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;

public class JCloudsObjectStoreFileStorage implements FileStorage {
import java.io.IOException;
import java.io.InputStream;
import java.text.MessageFormat;
import java.time.LocalDateTime;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;

public class JCloudsObjectStoreFileStorage extends ObjectStoreFileStorage {

private static final Logger LOGGER = LoggerFactory.getLogger(JCloudsObjectStoreFileStorage.class);
private static final int MAX_RETRIES_COUNT = 3;
Expand Down Expand Up @@ -75,6 +75,11 @@ public List<FileEntry> getFileEntriesWithoutContent(List<FileEntry> fileEntries)
.collect(Collectors.toList());
}

@Override
protected boolean existsInObjectStore(FileEntry fileEntry) {
return blobStore.blobMetadata(container, fileEntry.getId()) != null;
}

@Override
public void deleteFile(String id, String space) {
blobStore.removeBlob(container, id);
Expand Down
Loading
Loading