Skip to content

Commit f6c1e70

Browse files
IvanBorislavovDimitrovIvanBorislavovDimitrov
committed
Check if file exists in object store before reusing
LMCROSSITXSADEPLOY-3411
1 parent e6cb75f commit f6c1e70

15 files changed

Lines changed: 434 additions & 112 deletions

File tree

‎multiapps-controller-persistence/pom.xml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,15 @@
1414

1515
<build>
1616
<plugins>
17+
<plugin>
18+
<groupId>org.apache.maven.plugins</groupId>
19+
<artifactId>maven-surefire-plugin</artifactId>
20+
<configuration>
21+
<systemPropertyVariables>
22+
<java.util.logging.config.file>${project.basedir}/src/test/resources/logging.properties</java.util.logging.config.file>
23+
</systemPropertyVariables>
24+
</configuration>
25+
</plugin>
1726
<plugin>
1827
<groupId>de.empulse.eclipselink</groupId>
1928
<artifactId>staticweave-maven-plugin</artifactId>

‎multiapps-controller-persistence/src/main/java/module-info.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
requires google.cloud.nio;
4848
requires google.cloud.storage;
4949
requires jakarta.xml.bind;
50+
requires jakarta.annotation;
5051
requires jakarta.inject;
5152
requires liquibase.core;
5253
requires org.apache.logging.log4j;
@@ -59,6 +60,7 @@
5960
requires org.cloudfoundry.multiapps.common;
6061
requires org.eclipse.persistence.core;
6162
requires org.slf4j;
63+
requires spring.beans;
6264
requires spring.context;
6365
requires spring.core;
6466

‎multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,11 @@ public final class Messages {
88
// Exception messages:
99
public static final String FILE_UPLOAD_FAILED = "Upload of file \"{0}\" to \"{1}\" failed";
1010
public static final String FILE_NOT_FOUND = "File \"{0}\" not found";
11-
public static final String FAILED_TO_UPDATE_SQL_QUERY = "Failed to update SQL query";
1211
public static final String ERROR_FINDING_FILE_TO_UPLOAD = "Error finding file to upload with name {0}: {1}";
1312
public static final String ERROR_READING_FILE_CONTENT = "Error reading content of file {0}: {1}";
1413
public static final String FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST = "File with ID \"{0}\" and space \"{1}\" does not exist.";
1514
public static final String ERROR_GETTING_FILES_WITH_SPACE_AND_NAMESPACE = "Error getting files with space {0} and namespace {1}";
16-
public static final String ERROR_GETTING_FILES_WITH_SPACE_AND_OPERATION_ID = "Error getting files with space {0} and operation id {1}";
1715
public static final String ERROR_GETTING_LOGS_WITH_SPACE_AND_OPERATION_ID = "Error getting logs with space {0} and operation id {1}";
18-
public static final String ERROR_GETTING_FILES_WITH_SPACE_OPERATION_ID_AND_NAME = "Error getting files with space {0} operation id {1} and file name {2}";
1916
public static final String ERROR_GETTING_LOGS_WITH_SPACE_OPERATION_ID_AND_NAME = "Error getting logs with space {0} operation id {1} and file name {2}";
2017
public static final String ERROR_GETTING_ALL_FILES = "Error getting all files";
2118
public static final String ERROR_LOG_FILE_NOT_FOUND = "Log file with name \"{0}\" for operation \"{1}\" in space \"{2}\" was not found";
@@ -61,16 +58,12 @@ public final class Messages {
6158
public static final String COULD_NOT_CLOSE_RESULT_SET = "Could not close result set.";
6259
public static final String COULD_NOT_CLOSE_STATEMENT = "Could not close statement.";
6360
public static final String COULD_NOT_CLOSE_CONNECTION = "Could not close connection.";
64-
public static final String COULD_NOT_CLOSE_LOGGER_CONTEXT = "Could not close logger context";
65-
public static final String COULD_NOT_ROLLBACK_TRANSACTION = "Could not rollback transaction!";
66-
public static final String COULD_NOT_PERSIST_LOGS_FILE = "Could not persist logs file: {0}";
6761
public static final String ATTEMPT_TO_UPLOAD_BLOB_FAILED = "Attempt [{0}/{1}] to upload blob to ObjectStore failed with \"{2}\"";
6862
public static final String ATTEMPT_TO_DOWNLOAD_MISSING_BLOB = "Attempt [{0}/{1}] to download missing blob {2} from ObjectStore";
6963
public static final String USER_METADATA_OF_BLOB_0_EMPTY_AND_WILL_BE_DELETED = "User metadata of blob \"{0}\" is empty and will be deleted";
7064
public static final String DATE_METADATA_OF_BLOB_0_IS_NOT_IN_PROPER_FORMAT_AND_WILL_BE_DELETED = "Date metadata of blob \"{0}\" is not in a proper format and will be deleted";
7165

7266
// INFO log messages:
73-
public static final String DEFAULT_CONSOLE = "DefaultConsole";
7467
public static final String DELETING_FILES_WITHOUT_CONTENT_WITH_IDS_0 = "Deleting files without content with ids: {0}";
7568

7669
// DEBUG log messages:

‎multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java‎

Lines changed: 44 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,5 @@
11
package org.cloudfoundry.multiapps.controller.persistence.services;
22

3-
import java.io.IOException;
4-
import java.io.InputStream;
5-
import java.net.MalformedURLException;
6-
import java.net.URL;
7-
import java.time.LocalDateTime;
8-
import java.util.List;
9-
import java.util.Map;
10-
import java.util.Set;
11-
import java.util.function.Predicate;
12-
import java.util.stream.Collectors;
13-
143
import com.azure.core.http.HttpClient;
154
import com.azure.core.http.okhttp.OkHttpAsyncHttpClientBuilder;
165
import com.azure.core.http.policy.ExponentialBackoffOptions;
@@ -30,14 +19,31 @@
3019
import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants;
3120
import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter;
3221
import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper;
22+
import org.springframework.beans.factory.DisposableBean;
3323

34-
public class AzureObjectStoreFileStorage implements FileStorage {
24+
import java.io.IOException;
25+
import java.io.InputStream;
26+
import java.net.MalformedURLException;
27+
import java.net.URL;
28+
import java.time.LocalDateTime;
29+
import java.util.List;
30+
import java.util.Map;
31+
import java.util.Objects;
32+
import java.util.Set;
33+
import java.util.concurrent.CompletableFuture;
34+
import java.util.concurrent.ExecutorService;
35+
import java.util.concurrent.Executors;
36+
import java.util.function.Predicate;
37+
import java.util.stream.Collectors;
38+
39+
public class AzureObjectStoreFileStorage implements FileStorage, DisposableBean {
3540

3641
private static final String SAS_TOKEN = "sas_token";
3742
private static final String CONTAINER_NAME = "container_name";
3843
private static final String CONTAINER_URI = "container_uri";
3944
private final HttpClient httpClient;
4045
private final BlobContainerClient containerClient;
46+
private final ExecutorService virtualThreadExecutor = Executors.newVirtualThreadPerTaskExecutor();
4147

4248
public AzureObjectStoreFileStorage(Map<String, Object> credentials) {
4349
this.containerClient = createContainerClient(credentials);
@@ -66,6 +72,27 @@ public List<FileEntry> getFileEntriesWithoutContent(List<FileEntry> fileEntries)
6672
.toList();
6773
}
6874

75+
@Override
76+
public List<FileEntry> getExistingFileEntries(List<FileEntry> fileEntries) throws FileStorageException {
77+
if (fileEntries.isEmpty()) {
78+
return List.of();
79+
}
80+
List<CompletableFuture<FileEntry>> existenceChecks = fileEntries.stream()
81+
.map(fileEntry -> CompletableFuture.supplyAsync(
82+
() -> existsInBlobStore(fileEntry),
83+
virtualThreadExecutor))
84+
.toList();
85+
return existenceChecks.stream()
86+
.map(CompletableFuture::join)
87+
.filter(Objects::nonNull)
88+
.toList();
89+
}
90+
91+
private FileEntry existsInBlobStore(FileEntry fileEntry) {
92+
return containerClient.getBlobClient(fileEntry.getId())
93+
.exists() ? fileEntry : null;
94+
}
95+
6996
@Override
7097
public void deleteFile(String id, String space) throws FileStorageException {
7198
BlobClient blobClient = containerClient.getBlobClient(id);
@@ -210,4 +237,9 @@ public Set<String> getAllEntriesNames() {
210237
.map(BlobItem::getName)
211238
.collect(Collectors.toSet());
212239
}
240+
241+
@Override
242+
public void destroy() {
243+
virtualThreadExecutor.shutdown();
244+
}
213245
}

‎multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,20 @@
11
package org.cloudfoundry.multiapps.controller.persistence.services;
22

3-
import java.io.InputStream;
4-
import java.sql.SQLException;
5-
import java.time.LocalDateTime;
6-
import java.util.List;
7-
83
import org.cloudfoundry.multiapps.controller.persistence.Constants;
94
import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect;
5+
import org.cloudfoundry.multiapps.controller.persistence.Messages;
106
import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry;
117
import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry;
128
import org.cloudfoundry.multiapps.controller.persistence.query.options.StreamFetchingOptions;
139
import org.cloudfoundry.multiapps.controller.persistence.query.providers.BlobSqlFileQueryProvider;
1410
import org.cloudfoundry.multiapps.controller.persistence.query.providers.SqlFileQueryProvider;
1511

12+
import java.io.InputStream;
13+
import java.sql.SQLException;
14+
import java.text.MessageFormat;
15+
import java.time.LocalDateTime;
16+
import java.util.List;
17+
1618
public class DatabaseFileService extends FileService {
1719

1820
public DatabaseFileService(DataSourceWithDialect dataSourceWithDialect) {
@@ -27,15 +29,27 @@ protected DatabaseFileService(DataSourceWithDialect dataSourceWithDialect, SqlFi
2729
super(dataSourceWithDialect, sqlFileQueryProvider, null);
2830
}
2931

32+
@Override
33+
public List<FileEntry> listFiles(String space, String namespace) throws FileStorageException {
34+
try {
35+
return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getListFilesQuery(space, namespace));
36+
} catch (SQLException e) {
37+
throw new FileStorageException(MessageFormat.format(Messages.ERROR_GETTING_FILES_WITH_SPACE_AND_NAMESPACE, space, namespace),
38+
e);
39+
}
40+
}
41+
3042
@Override
3143
public <T> T processFileContentWithOffset(FileContentToProcess fileContentToProcess, FileContentProcessor<T> fileContentProcessor)
3244
throws FileStorageException {
3345
try {
34-
return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getProcessFileWithContentQueryWithOffsetQuery(fileContentToProcess.getSpaceGuid(),
35-
fileContentToProcess.getGuid(),
36-
new StreamFetchingOptions(fileContentToProcess.getStartOffset(),
37-
fileContentToProcess.getEndOffset()),
38-
fileContentProcessor));
46+
return getSqlQueryExecutor().execute(
47+
getSqlFileQueryProvider().getProcessFileWithContentQueryWithOffsetQuery(fileContentToProcess.getSpaceGuid(),
48+
fileContentToProcess.getGuid(),
49+
new StreamFetchingOptions(
50+
fileContentToProcess.getStartOffset(),
51+
fileContentToProcess.getEndOffset()),
52+
fileContentProcessor));
3953
} catch (SQLException e) {
4054
throw new FileStorageException(e.getMessage(), e);
4155
}

‎multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java‎

Lines changed: 15 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,17 @@
11
package org.cloudfoundry.multiapps.controller.persistence.services;
22

3+
import jakarta.xml.bind.DatatypeConverter;
4+
import org.cloudfoundry.multiapps.controller.persistence.Constants;
5+
import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect;
6+
import org.cloudfoundry.multiapps.controller.persistence.Messages;
7+
import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry;
8+
import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry;
9+
import org.cloudfoundry.multiapps.controller.persistence.query.providers.ExternalSqlFileQueryProvider;
10+
import org.cloudfoundry.multiapps.controller.persistence.query.providers.SqlFileQueryProvider;
11+
import org.cloudfoundry.multiapps.controller.persistence.util.SqlQueryExecutor;
12+
import org.slf4j.Logger;
13+
import org.slf4j.LoggerFactory;
14+
315
import java.io.BufferedInputStream;
416
import java.io.File;
517
import java.io.FileInputStream;
@@ -16,19 +28,6 @@
1628
import java.util.List;
1729
import java.util.UUID;
1830

19-
import org.cloudfoundry.multiapps.controller.persistence.Constants;
20-
import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect;
21-
import org.cloudfoundry.multiapps.controller.persistence.Messages;
22-
import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry;
23-
import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry;
24-
import org.cloudfoundry.multiapps.controller.persistence.query.providers.ExternalSqlFileQueryProvider;
25-
import org.cloudfoundry.multiapps.controller.persistence.query.providers.SqlFileQueryProvider;
26-
import org.cloudfoundry.multiapps.controller.persistence.util.SqlQueryExecutor;
27-
import org.slf4j.Logger;
28-
import org.slf4j.LoggerFactory;
29-
30-
import jakarta.xml.bind.DatatypeConverter;
31-
3231
public class FileService {
3332

3433
protected static final String DEFAULT_TABLE_NAME = "LM_SL_PERSISTENCE_FILE";
@@ -77,23 +76,15 @@ public FileEntry addFile(FileEntry fileEntry, File existingFile) throws FileStor
7776

7877
public List<FileEntry> listFiles(String space, String namespace) throws FileStorageException {
7978
try {
80-
return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getListFilesQuery(space, namespace));
79+
List<FileEntry> fileEntriesFromDb = getSqlQueryExecutor().execute(
80+
getSqlFileQueryProvider().getListFilesQuery(space, namespace));
81+
return fileStorage.getExistingFileEntries(fileEntriesFromDb);
8182
} catch (SQLException e) {
8283
throw new FileStorageException(MessageFormat.format(Messages.ERROR_GETTING_FILES_WITH_SPACE_AND_NAMESPACE, space, namespace),
8384
e);
8485
}
8586
}
8687

87-
public List<FileEntry> listFilesBySpaceAndOperationId(String space, String operationId) throws FileStorageException {
88-
try {
89-
return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getListFilesBySpaceAndOperationId(space, operationId));
90-
} catch (SQLException e) {
91-
throw new FileStorageException(MessageFormat.format(Messages.ERROR_GETTING_FILES_WITH_SPACE_AND_OPERATION_ID, space,
92-
operationId),
93-
e);
94-
}
95-
}
96-
9788
public List<FileEntry> listFilesCreatedAfterAndBeforeWithoutOperationId(LocalDateTime after, LocalDateTime before)
9889
throws FileStorageException {
9990
try {

‎multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ public interface FileStorage {
1313
@Deprecated // This method is not reliable for aws as BlobStore::list might not return a complete list
1414
List<FileEntry> getFileEntriesWithoutContent(List<FileEntry> fileEntries) throws FileStorageException;
1515

16+
List<FileEntry> getExistingFileEntries(List<FileEntry> fileEntries) throws FileStorageException;
17+
1618
void deleteFile(String id, String space) throws FileStorageException;
1719

1820
void deleteFilesBySpaceIds(List<String> spaceIds) throws FileStorageException;

‎multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java‎

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,5 @@
11
package org.cloudfoundry.multiapps.controller.persistence.services;
22

3-
import java.io.ByteArrayInputStream;
4-
import java.io.IOException;
5-
import java.io.InputStream;
6-
import java.nio.channels.Channels;
7-
import java.text.MessageFormat;
8-
import java.time.LocalDateTime;
9-
import java.util.ArrayList;
10-
import java.util.Base64;
11-
import java.util.List;
12-
import java.util.Map;
13-
import java.util.Set;
14-
import java.util.function.Predicate;
15-
import java.util.stream.Collectors;
16-
173
import com.google.api.gax.retrying.RetrySettings;
184
import com.google.auth.Credentials;
195
import com.google.auth.oauth2.GoogleCredentials;
@@ -32,6 +18,21 @@
3218
import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper;
3319
import org.springframework.http.MediaType;
3420

21+
import java.io.ByteArrayInputStream;
22+
import java.io.IOException;
23+
import java.io.InputStream;
24+
import java.nio.channels.Channels;
25+
import java.text.MessageFormat;
26+
import java.time.LocalDateTime;
27+
import java.util.ArrayList;
28+
import java.util.Base64;
29+
import java.util.List;
30+
import java.util.Map;
31+
import java.util.Objects;
32+
import java.util.Set;
33+
import java.util.function.Predicate;
34+
import java.util.stream.Collectors;
35+
3536
public class GcpObjectStoreFileStorage implements FileStorage {
3637

3738
private final String bucketName;
@@ -104,6 +105,24 @@ public List<FileEntry> getFileEntriesWithoutContent(List<FileEntry> fileEntries)
104105
.toList();
105106
}
106107

108+
@Override
109+
public List<FileEntry> getExistingFileEntries(List<FileEntry> fileEntries) {
110+
if (fileEntries.isEmpty()) {
111+
return List.of();
112+
}
113+
List<BlobId> blobIds = fileEntries.stream()
114+
.map(fileEntry -> BlobId.of(bucketName, fileEntry.getId()))
115+
.toList();
116+
List<Blob> blobs = storage.get(blobIds);
117+
Set<String> existingBlobNames = blobs.stream()
118+
.filter(Objects::nonNull)
119+
.map(Blob::getName)
120+
.collect(Collectors.toSet());
121+
return fileEntries.stream()
122+
.filter(fileEntry -> existingBlobNames.contains(fileEntry.getId()))
123+
.toList();
124+
}
125+
107126
@Override
108127
public void deleteFile(String id, String space) {
109128
deleteFileWithGeneration(id);

0 commit comments

Comments
 (0)