diff --git a/multiapps-controller-persistence/pom.xml b/multiapps-controller-persistence/pom.xml index 2a9d41446e..aee4f64791 100644 --- a/multiapps-controller-persistence/pom.xml +++ b/multiapps-controller-persistence/pom.xml @@ -14,6 +14,15 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + + ${project.basedir}/src/test/resources/logging.properties + + + de.empulse.eclipselink staticweave-maven-plugin diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index 7c49523d76..20b366fbd9 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -47,6 +47,7 @@ requires google.cloud.nio; requires google.cloud.storage; requires jakarta.xml.bind; + requires jakarta.annotation; requires jakarta.inject; requires liquibase.core; requires org.apache.logging.log4j; @@ -59,6 +60,7 @@ requires org.cloudfoundry.multiapps.common; requires org.eclipse.persistence.core; requires org.slf4j; + requires spring.beans; requires spring.context; requires spring.core; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java index d255130db0..5bd5769e19 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java @@ -8,14 +8,11 @@ public final class Messages { // Exception messages: public static final String FILE_UPLOAD_FAILED = "Upload of file \"{0}\" to \"{1}\" failed"; public static final String FILE_NOT_FOUND = "File \"{0}\" not found"; - public static final String FAILED_TO_UPDATE_SQL_QUERY = "Failed to update SQL query"; public static final String ERROR_FINDING_FILE_TO_UPLOAD = "Error finding file to upload with name {0}: {1}"; public static final String ERROR_READING_FILE_CONTENT = "Error reading content of file {0}: {1}"; public static final String FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST = "File with ID \"{0}\" and space \"{1}\" does not exist."; public static final String ERROR_GETTING_FILES_WITH_SPACE_AND_NAMESPACE = "Error getting files with space {0} and namespace {1}"; - public static final String ERROR_GETTING_FILES_WITH_SPACE_AND_OPERATION_ID = "Error getting files with space {0} and operation id {1}"; public static final String ERROR_GETTING_LOGS_WITH_SPACE_AND_OPERATION_ID = "Error getting logs with space {0} and operation id {1}"; - public static final String ERROR_GETTING_FILES_WITH_SPACE_OPERATION_ID_AND_NAME = "Error getting files with space {0} operation id {1} and file name {2}"; public static final String ERROR_GETTING_LOGS_WITH_SPACE_OPERATION_ID_AND_NAME = "Error getting logs with space {0} operation id {1} and file name {2}"; public static final String ERROR_GETTING_ALL_FILES = "Error getting all files"; public static final String ERROR_LOG_FILE_NOT_FOUND = "Log file with name \"{0}\" for operation \"{1}\" in space \"{2}\" was not found"; @@ -61,16 +58,12 @@ public final class Messages { public static final String COULD_NOT_CLOSE_RESULT_SET = "Could not close result set."; public static final String COULD_NOT_CLOSE_STATEMENT = "Could not close statement."; public static final String COULD_NOT_CLOSE_CONNECTION = "Could not close connection."; - public static final String COULD_NOT_CLOSE_LOGGER_CONTEXT = "Could not close logger context"; - public static final String COULD_NOT_ROLLBACK_TRANSACTION = "Could not rollback transaction!"; - public static final String COULD_NOT_PERSIST_LOGS_FILE = "Could not persist logs file: {0}"; public static final String ATTEMPT_TO_UPLOAD_BLOB_FAILED = "Attempt [{0}/{1}] to upload blob to ObjectStore failed with \"{2}\""; public static final String ATTEMPT_TO_DOWNLOAD_MISSING_BLOB = "Attempt [{0}/{1}] to download missing blob {2} from ObjectStore"; public static final String USER_METADATA_OF_BLOB_0_EMPTY_AND_WILL_BE_DELETED = "User metadata of blob \"{0}\" is empty and will be deleted"; public static final String DATE_METADATA_OF_BLOB_0_IS_NOT_IN_PROPER_FORMAT_AND_WILL_BE_DELETED = "Date metadata of blob \"{0}\" is not in a proper format and will be deleted"; // INFO log messages: - public static final String DEFAULT_CONSOLE = "DefaultConsole"; public static final String DELETING_FILES_WITHOUT_CONTENT_WITH_IDS_0 = "Deleting files without content with ids: {0}"; // DEBUG log messages: diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java index e3da858584..8a454dfb1a 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java @@ -1,16 +1,5 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URL; -import java.time.LocalDateTime; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - import com.azure.core.http.HttpClient; import com.azure.core.http.okhttp.OkHttpAsyncHttpClientBuilder; import com.azure.core.http.policy.ExponentialBackoffOptions; @@ -30,8 +19,18 @@ import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; -public class AzureObjectStoreFileStorage implements FileStorage { +public class AzureObjectStoreFileStorage extends ObjectStoreFileStorage { private static final String SAS_TOKEN = "sas_token"; private static final String CONTAINER_NAME = "container_name"; @@ -66,6 +65,12 @@ public List getFileEntriesWithoutContent(List fileEntries) .toList(); } + @Override + protected boolean existsInObjectStore(FileEntry fileEntry) { + return containerClient.getBlobClient(fileEntry.getId()) + .exists(); + } + @Override public void deleteFile(String id, String space) throws FileStorageException { BlobClient blobClient = containerClient.getBlobClient(id); diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java index 7a6586710a..5895b9fe32 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java @@ -1,18 +1,20 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import java.io.InputStream; -import java.sql.SQLException; -import java.time.LocalDateTime; -import java.util.List; - import org.cloudfoundry.multiapps.controller.persistence.Constants; import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect; +import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; import org.cloudfoundry.multiapps.controller.persistence.query.options.StreamFetchingOptions; import org.cloudfoundry.multiapps.controller.persistence.query.providers.BlobSqlFileQueryProvider; import org.cloudfoundry.multiapps.controller.persistence.query.providers.SqlFileQueryProvider; +import java.io.InputStream; +import java.sql.SQLException; +import java.text.MessageFormat; +import java.time.LocalDateTime; +import java.util.List; + public class DatabaseFileService extends FileService { public DatabaseFileService(DataSourceWithDialect dataSourceWithDialect) { @@ -27,15 +29,27 @@ protected DatabaseFileService(DataSourceWithDialect dataSourceWithDialect, SqlFi super(dataSourceWithDialect, sqlFileQueryProvider, null); } + @Override + public List listFiles(String space, String namespace) throws FileStorageException { + try { + return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getListFilesQuery(space, namespace)); + } catch (SQLException e) { + throw new FileStorageException(MessageFormat.format(Messages.ERROR_GETTING_FILES_WITH_SPACE_AND_NAMESPACE, space, namespace), + e); + } + } + @Override public T processFileContentWithOffset(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) throws FileStorageException { try { - return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getProcessFileWithContentQueryWithOffsetQuery(fileContentToProcess.getSpaceGuid(), - fileContentToProcess.getGuid(), - new StreamFetchingOptions(fileContentToProcess.getStartOffset(), - fileContentToProcess.getEndOffset()), - fileContentProcessor)); + return getSqlQueryExecutor().execute( + getSqlFileQueryProvider().getProcessFileWithContentQueryWithOffsetQuery(fileContentToProcess.getSpaceGuid(), + fileContentToProcess.getGuid(), + new StreamFetchingOptions( + fileContentToProcess.getStartOffset(), + fileContentToProcess.getEndOffset()), + fileContentProcessor)); } catch (SQLException e) { throw new FileStorageException(e.getMessage(), e); } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java index b2d749dc6c..340c6734ab 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java @@ -1,5 +1,17 @@ package org.cloudfoundry.multiapps.controller.persistence.services; +import jakarta.xml.bind.DatatypeConverter; +import org.cloudfoundry.multiapps.controller.persistence.Constants; +import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect; +import org.cloudfoundry.multiapps.controller.persistence.Messages; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.query.providers.ExternalSqlFileQueryProvider; +import org.cloudfoundry.multiapps.controller.persistence.query.providers.SqlFileQueryProvider; +import org.cloudfoundry.multiapps.controller.persistence.util.SqlQueryExecutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; @@ -16,19 +28,6 @@ import java.util.List; import java.util.UUID; -import org.cloudfoundry.multiapps.controller.persistence.Constants; -import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect; -import org.cloudfoundry.multiapps.controller.persistence.Messages; -import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; -import org.cloudfoundry.multiapps.controller.persistence.query.providers.ExternalSqlFileQueryProvider; -import org.cloudfoundry.multiapps.controller.persistence.query.providers.SqlFileQueryProvider; -import org.cloudfoundry.multiapps.controller.persistence.util.SqlQueryExecutor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import jakarta.xml.bind.DatatypeConverter; - public class FileService { protected static final String DEFAULT_TABLE_NAME = "LM_SL_PERSISTENCE_FILE"; @@ -77,23 +76,15 @@ public FileEntry addFile(FileEntry fileEntry, File existingFile) throws FileStor public List listFiles(String space, String namespace) throws FileStorageException { try { - return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getListFilesQuery(space, namespace)); + List fileEntriesFromDb = getSqlQueryExecutor().execute( + getSqlFileQueryProvider().getListFilesQuery(space, namespace)); + return fileStorage.getExistingFileEntries(fileEntriesFromDb); } catch (SQLException e) { throw new FileStorageException(MessageFormat.format(Messages.ERROR_GETTING_FILES_WITH_SPACE_AND_NAMESPACE, space, namespace), e); } } - public List listFilesBySpaceAndOperationId(String space, String operationId) throws FileStorageException { - try { - return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getListFilesBySpaceAndOperationId(space, operationId)); - } catch (SQLException e) { - throw new FileStorageException(MessageFormat.format(Messages.ERROR_GETTING_FILES_WITH_SPACE_AND_OPERATION_ID, space, - operationId), - e); - } - } - public List listFilesCreatedAfterAndBeforeWithoutOperationId(LocalDateTime after, LocalDateTime before) throws FileStorageException { try { diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java index 086e713708..0cdca9cf4f 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java @@ -13,6 +13,8 @@ public interface FileStorage { @Deprecated // This method is not reliable for aws as BlobStore::list might not return a complete list List getFileEntriesWithoutContent(List fileEntries) throws FileStorageException; + List getExistingFileEntries(List fileEntries) throws FileStorageException; + void deleteFile(String id, String space) throws FileStorageException; void deleteFilesBySpaceIds(List spaceIds) throws FileStorageException; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index 153a734814..a314d15d7b 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -1,19 +1,5 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.Channels; -import java.text.MessageFormat; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - import com.google.api.gax.retrying.RetrySettings; import com.google.auth.Credentials; import com.google.auth.oauth2.GoogleCredentials; @@ -32,6 +18,21 @@ import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.springframework.http.MediaType; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.text.MessageFormat; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + public class GcpObjectStoreFileStorage implements FileStorage { private final String bucketName; @@ -104,6 +105,24 @@ public List getFileEntriesWithoutContent(List fileEntries) .toList(); } + @Override + public List getExistingFileEntries(List fileEntries) { + if (fileEntries.isEmpty()) { + return List.of(); + } + List blobIds = fileEntries.stream() + .map(fileEntry -> BlobId.of(bucketName, fileEntry.getId())) + .toList(); + List blobs = storage.get(blobIds); + Set existingBlobNames = blobs.stream() + .filter(Objects::nonNull) + .map(Blob::getName) + .collect(Collectors.toSet()); + return fileEntries.stream() + .filter(fileEntry -> existingBlobNames.contains(fileEntry.getId())) + .toList(); + } + @Override public void deleteFile(String id, String space) { deleteFileWithGeneration(id); diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java index 2e5ec53acb..d1fba0ac70 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java @@ -1,16 +1,5 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import java.io.IOException; -import java.io.InputStream; -import java.text.MessageFormat; -import java.time.LocalDateTime; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - import org.cloudfoundry.multiapps.common.util.MiscUtil; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; @@ -30,7 +19,18 @@ import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; -public class JCloudsObjectStoreFileStorage implements FileStorage { +import java.io.IOException; +import java.io.InputStream; +import java.text.MessageFormat; +import java.time.LocalDateTime; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public class JCloudsObjectStoreFileStorage extends ObjectStoreFileStorage { private static final Logger LOGGER = LoggerFactory.getLogger(JCloudsObjectStoreFileStorage.class); private static final int MAX_RETRIES_COUNT = 3; @@ -75,6 +75,11 @@ public List getFileEntriesWithoutContent(List fileEntries) .collect(Collectors.toList()); } + @Override + protected boolean existsInObjectStore(FileEntry fileEntry) { + return blobStore.blobMetadata(container, fileEntry.getId()) != null; + } + @Override public void deleteFile(String id, String space) { blobStore.removeBlob(container, id); diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java new file mode 100644 index 0000000000..13ec8944dd --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java @@ -0,0 +1,47 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.springframework.beans.factory.DisposableBean; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public abstract class ObjectStoreFileStorage implements FileStorage, DisposableBean { + + private final ExecutorService virtualThreadExecutor = Executors.newVirtualThreadPerTaskExecutor(); + + @Override + public List getExistingFileEntries(List fileEntries) { + if (fileEntries.isEmpty()) { + return List.of(); + } + List> existenceChecks = fileEntries.stream() + .map(this::asyncCheckExistenceOfFileEntry) + .toList(); + return existenceChecks.stream() + .map(CompletableFuture::join) + .filter(Objects::nonNull) + .toList(); + } + + private CompletableFuture asyncCheckExistenceOfFileEntry(FileEntry fileEntry) { + return CompletableFuture.supplyAsync(() -> toFileEntryIfExists(fileEntry), virtualThreadExecutor); + } + + private FileEntry toFileEntryIfExists(FileEntry fileEntry) { + if (existsInObjectStore(fileEntry)) { + return fileEntry; + } + return null; + } + + protected abstract boolean existsInObjectStore(FileEntry fileEntry); + + @Override + public void destroy() { + virtualThreadExecutor.shutdown(); + } +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java index 437981fa8a..1200462686 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java @@ -27,9 +27,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -260,6 +262,49 @@ void testDeleteFilesByIds() throws FileStorageException { verify(blobClient).deleteIfExists(); } + @Test + void getExistingFileEntriesWhenAllEntriesExist() throws FileStorageException { + when(blobClient.exists()).thenReturn(true); + FileEntry firstEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + FileEntry secondEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); + + List result = fileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + + assertEquals(2, result.size()); + List returnedIds = result.stream() + .map(FileEntry::getId) + .toList(); + assertTrue(returnedIds.contains(TEST_ID)); + assertTrue(returnedIds.contains(TEST_ID_2)); + } + + @Test + void getExistingFileEntriesWhenNoEntriesExist() throws FileStorageException { + when(blobClient.exists()).thenReturn(false); + FileEntry firstEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + FileEntry secondEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); + + List result = fileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + + assertTrue(result.isEmpty()); + } + + @Test + void getExistingFileEntriesWhenSomeEntriesExist() throws FileStorageException { + when(blobContainerClient.getBlobClient(TEST_ID)).thenReturn(blobClient); + BlobClient nonExistingBlobClient = mock(BlobClient.class); + when(blobContainerClient.getBlobClient(TEST_ID_2)).thenReturn(nonExistingBlobClient); + when(blobClient.exists()).thenReturn(true); + when(nonExistingBlobClient.exists()).thenReturn(false); + FileEntry existingEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + FileEntry nonExistingEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); + + List result = fileStorage.getExistingFileEntries(List.of(existingEntry, nonExistingEntry)); + + assertEquals(1, result.size()); + assertEquals(TEST_ID, result.getFirst().getId()); + } + private void setupDeleteMethods(BlobItem... blobItems) { when(pagedIterable.stream()).thenReturn(Stream.of(blobItems)); when(blobContainerClient.listBlobs(any(), any())).thenReturn(pagedIterable); diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/FileServiceTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/FileServiceTest.java index 8e38f41a67..483c179b5b 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/FileServiceTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/FileServiceTest.java @@ -40,6 +40,8 @@ public void setUp() throws Exception { Mockito.doAnswer(invocationOnMock -> IOUtils.consume((InputStream) invocationOnMock.getArgument(1))) .when(fileStorage) .addFile(Mockito.any(), Mockito.any()); + Mockito.when(fileStorage.getExistingFileEntries(Mockito.anyList())) + .thenAnswer(invocation -> invocation.getArgument(0)); } @Test @@ -152,6 +154,56 @@ void testOpenInputStream() throws Exception { .openInputStream(anyString(), anyString()); } + @Test + void listFilesReturnsOnlyEntriesExistingInObjectStore() throws Exception { + FileEntry existingInBoth = addTestFile(SPACE_1, NAMESPACE_1); + addTestFile(SPACE_1, NAMESPACE_1); // exists in DB but not in object store + + Mockito.when(fileStorage.getExistingFileEntries(Mockito.anyList())) + .thenAnswer(invocation -> { + List entries = invocation.getArgument(0); + return entries.stream() + .filter(entry -> entry.getId() + .equals(existingInBoth.getId())) + .toList(); + }); + + List result = fileService.listFiles(SPACE_1, NAMESPACE_1); + + assertEquals(1, result.size()); + assertEquals(existingInBoth.getId(), result.get(0) + .getId()); + } + + @Test + void listFilesReturnsAllEntriesWhenAllExistInObjectStore() throws Exception { + FileEntry fileEntry1 = addTestFile(SPACE_1, NAMESPACE_1); + FileEntry fileEntry2 = addTestFile(SPACE_1, NAMESPACE_1); + + List result = fileService.listFiles(SPACE_1, NAMESPACE_1); + + assertEquals(2, result.size()); + assertTrue(result.stream() + .anyMatch(entry -> entry.getId() + .equals(fileEntry1.getId()))); + assertTrue(result.stream() + .anyMatch(entry -> entry.getId() + .equals(fileEntry2.getId()))); + } + + @Test + void listFilesReturnsEmptyListWhenNoEntriesExistInObjectStore() throws Exception { + addTestFile(SPACE_1, NAMESPACE_1); + addTestFile(SPACE_1, NAMESPACE_1); + + Mockito.when(fileStorage.getExistingFileEntries(Mockito.anyList())) + .thenReturn(List.of()); + + List result = fileService.listFiles(SPACE_1, NAMESPACE_1); + + assertEquals(0, result.size()); + } + @Test void deleteFilesEntriesWithoutContentTest() throws Exception { FileEntry noContent = addTestFile(SPACE_1, NAMESPACE_1); diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java index ab09be0e9c..3d2cfeeb19 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -1,27 +1,38 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; -import java.util.UUID; - import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.springframework.http.MediaType; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; class GcpObjectStoreFileStorageTest extends JCloudsObjectStoreFileStorageTest { private Storage storage; + private Storage mockedStorage; + private GcpObjectStoreFileStorage mockedGcpFileStorage; @Override @BeforeEach @@ -39,6 +50,13 @@ protected Storage createObjectStoreStorage(Map credentials) { .toString(); namespace = UUID.randomUUID() .toString(); + mockedStorage = mock(Storage.class); + mockedGcpFileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { + @Override + protected Storage createObjectStoreStorage(Map credentials) { + return mockedStorage; + } + }; } @Override @@ -80,4 +98,76 @@ public void assertFileExists(boolean exceptedFileExist, FileEntry actualFile) { assertEquals(exceptedFileExist, blobExists); } + @Override + @Test + void getExistingFileEntriesAllExist() { + FileEntry firstEntry = createFileEntryWithRandomId(); + FileEntry secondEntry = createFileEntryWithRandomId(); + mockStorageGetToReturn(List.of(blobWithName(firstEntry.getId()), blobWithName(secondEntry.getId()))); + + List result = mockedGcpFileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + + assertEquals(2, result.size()); + List returnedIds = result.stream() + .map(FileEntry::getId) + .toList(); + assertTrue(returnedIds.contains(firstEntry.getId())); + assertTrue(returnedIds.contains(secondEntry.getId())); + } + + @Override + @Test + void getExistingFileEntriesNoneExist() { + FileEntry firstEntry = createFileEntryWithRandomId(); + FileEntry secondEntry = createFileEntryWithRandomId(); + mockStorageGetToReturn(Arrays.asList(null, null)); + + List result = mockedGcpFileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + + assertTrue(result.isEmpty()); + } + + @Override + @Test + void getExistingFileEntriesSomeExist() { + FileEntry existingEntry = createFileEntryWithRandomId(); + FileEntry nonExistingEntry = createFileEntryWithRandomId(); + mockStorageGetToReturn(Arrays.asList(blobWithName(existingEntry.getId()), null)); + + List result = mockedGcpFileStorage.getExistingFileEntries(List.of(existingEntry, nonExistingEntry)); + + assertEquals(1, result.size()); + assertEquals(existingEntry.getId(), result.getFirst() + .getId()); + } + + @Test + void getExistingFileEntriesPassesCorrectBlobIdsToStorage() { + FileEntry entry = createFileEntryWithRandomId(); + mockStorageGetToReturn(List.of()); + + mockedGcpFileStorage.getExistingFileEntries(List.of(entry)); + + verify(mockedStorage).get(List.of(BlobId.of(CONTAINER, entry.getId()))); + } + + private void mockStorageGetToReturn(List blobs) { + when(mockedStorage.get(anyList())).thenReturn(blobs); + } + + private FileEntry createFileEntryWithRandomId() { + return ImmutableFileEntry.builder() + .id(UUID.randomUUID() + .toString()) + .space(spaceId) + .namespace(namespace) + .build(); + } + + private Blob blobWithName(String name) { + Blob blob = mock(Blob.class); + when(blob.getName()).thenReturn(name); + return blob; + } + } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java index 6356afd641..29a5398b45 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java @@ -1,5 +1,18 @@ package org.cloudfoundry.multiapps.controller.persistence.services; +import jakarta.xml.bind.DatatypeConverter; +import org.cloudfoundry.multiapps.common.util.DigestHelper; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.Blob; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.MediaType; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -16,23 +29,11 @@ import java.util.List; import java.util.UUID; -import jakarta.xml.bind.DatatypeConverter; -import org.cloudfoundry.multiapps.common.util.DigestHelper; -import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; -import org.jclouds.ContextBuilder; -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.domain.Blob; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.http.MediaType; - import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class JCloudsObjectStoreFileStorageTest { @@ -170,6 +171,52 @@ protected void assertBlobDoesNotExist(String blobWithNoMetadataId) { .getBlob(CONTAINER, blobWithNoMetadataId)); } + @Test + void getExistingFileEntriesAllExist() throws Exception { + FileEntry firstFile = addFile(TEST_FILE_LOCATION); + FileEntry secondFile = addFile(SECOND_FILE_TEST_LOCATION); + + List existingEntries = fileStorage.getExistingFileEntries(List.of(firstFile, secondFile)); + + assertEquals(2, existingEntries.size()); + List returnedIds = existingEntries.stream() + .map(FileEntry::getId) + .toList(); + assertTrue(returnedIds.contains(firstFile.getId())); + assertTrue(returnedIds.contains(secondFile.getId())); + } + + @Test + void getExistingFileEntriesNoneExist() throws FileStorageException { + FileEntry nonExistingFile1 = createFileEntryWithRandomId(); + FileEntry nonExistingFile2 = createFileEntryWithRandomId(); + + List existingEntries = fileStorage.getExistingFileEntries(List.of(nonExistingFile1, nonExistingFile2)); + + assertTrue(existingEntries.isEmpty()); + } + + @Test + void getExistingFileEntriesSomeExist() throws Exception { + FileEntry existingFile = addFile(TEST_FILE_LOCATION); + FileEntry nonExistingFile = createFileEntryWithRandomId(); + + List existingEntries = fileStorage.getExistingFileEntries(List.of(existingFile, nonExistingFile)); + + assertEquals(1, existingEntries.size()); + assertEquals(existingFile.getId(), existingEntries.get(0) + .getId()); + } + + private FileEntry createFileEntryWithRandomId() { + return ImmutableFileEntry.builder() + .id(UUID.randomUUID() + .toString()) + .space(spaceId) + .namespace(namespace) + .build(); + } + @Test void testConnection() { assertDoesNotThrow(() -> fileStorage.testConnection()); diff --git a/multiapps-controller-persistence/src/test/resources/logging.properties b/multiapps-controller-persistence/src/test/resources/logging.properties new file mode 100644 index 0000000000..2d604d8415 --- /dev/null +++ b/multiapps-controller-persistence/src/test/resources/logging.properties @@ -0,0 +1,2 @@ +com.google.api.client.googleapis.services.AbstractGoogleClient.level=SEVERE + diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationInFinalStateHandler.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationInFinalStateHandler.java index 475ae33eba..87f8f900bd 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationInFinalStateHandler.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationInFinalStateHandler.java @@ -1,12 +1,5 @@ package org.cloudfoundry.multiapps.controller.process.util; -import java.text.MessageFormat; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; - import jakarta.inject.Inject; import jakarta.inject.Named; import org.cloudfoundry.client.v3.Metadata; @@ -41,6 +34,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.text.MessageFormat; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + import static java.text.MessageFormat.format; @Named diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStepTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStepTest.java index 168e3c9821..8956cc1873 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStepTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStepTest.java @@ -1,18 +1,5 @@ package org.cloudfoundry.multiapps.controller.process.steps; -import java.io.InputStream; -import java.math.BigInteger; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.text.MessageFormat; -import java.time.Duration; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.FutureTask; -import java.util.stream.Stream; - import org.cloudfoundry.multiapps.common.SLException; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.validators.parameters.FileMimeTypeValidator; @@ -28,6 +15,18 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mockito; +import java.io.InputStream; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.MessageFormat; +import java.time.Duration; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.FutureTask; +import java.util.stream.Stream; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -113,7 +112,7 @@ private void initializeComponents(StepInput stepInput, boolean isArchiveChunked) this.stepInput = stepInput; this.isArchiveChunked = isArchiveChunked; prepareContext(); - prepareFileService(stepInput.appArchiveId); + prepareFileService(); prepareArchiveMerger(); prepareConfiguration(); } @@ -128,7 +127,7 @@ private void prepareContext() { context.setVariable(Variables.MTA_NAMESPACE, "namespace"); } - private void prepareFileService(String appArchiveId) throws FileStorageException { + private void prepareFileService() throws FileStorageException { when(fileService.getFile("space-id", EXISTING_FILE_ID)) .thenReturn(createFileEntry(EXISTING_FILE_ID, "some-file-entry-name", 1024 * 1024L)); when(fileService.getFile("space-id", MERGED_ARCHIVE_NAME + ".part.0")) @@ -161,20 +160,6 @@ private void prepareFileService(String appArchiveId) throws FileStorageException .thenReturn(null); when(fileService.addFile(any(FileEntry.class), any(InputStream.class))) .thenReturn(createFileEntry(EXISTING_FILE_ID, MERGED_ARCHIVE_TEST_MTAR, 1024 * 1024 * 1024L)); - if (appArchiveId.contains(EXCEEDING_FILE_SIZE_ID)) { - List fileEntries = List.of(createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.0", EXCEEDING_FILE_SIZE_ID + ".part.0", - 1024 * 1024 * 1024), - createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.1", EXCEEDING_FILE_SIZE_ID + ".part.1", - 1024 * 1024 * 1024), - createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.2", EXCEEDING_FILE_SIZE_ID + ".part.2", - 1024 * 1024 * 1024), - createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.3", EXCEEDING_FILE_SIZE_ID + ".part.3", - 1024 * 1024 * 1024), - createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.4", EXCEEDING_FILE_SIZE_ID + ".part.4", - 1024 * 1024 * 1024)); - when(fileService.listFilesBySpaceAndOperationId(Mockito.anyString(), Mockito.anyString())) - .thenReturn(fileEntries); - } } private void prepareArchiveMerger() {