From 01961a5ba1db4084c577794138cb7ab5c3a63b90 Mon Sep 17 00:00:00 2001 From: Sergey Soldatov Date: Thu, 21 May 2026 12:59:25 -0700 Subject: [PATCH] HDDS-13855. Add integration tests for ACL enforcement after leadership change Add HA integration tests to TestOMHALeaderSpecificACLEnforcement covering ACL enforcement in preExecute for: - Volume operations: delete, set-owner, set-quota - Bucket operations: delete, set-owner, set-property - Key bulk operations: delete-keys, rename-keys - Key ACL operations: add/remove/set ACL (FSO layout) Each test triggers a leadership change and verifies that unauthorized requests are rejected before a Ratis log entry is written, confirming that the ACL check runs in preExecute on the new leader. Co-authored-by: Cursor --- .../TestOMHALeaderSpecificACLEnforcement.java | 695 +++++++++++++++++- 1 file changed, 680 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMHALeaderSpecificACLEnforcement.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMHALeaderSpecificACLEnforcement.java index 43acb0f823d9..86d2fe66137a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMHALeaderSpecificACLEnforcement.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMHALeaderSpecificACLEnforcement.java @@ -22,22 +22,31 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_DELETE; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PERMISSION_DENIED; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.concurrent.TimeoutException; +import java.util.function.BooleanSupplier; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.hdds.client.OzoneQuota; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; +import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -48,8 +57,11 @@ import org.apache.hadoop.ozone.client.VolumeArgs; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer; +import org.apache.hadoop.ozone.security.acl.OzoneObj; +import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterAll; @@ -111,14 +123,14 @@ public void restoreLeadership() throws IOException, InterruptedException, Timeou OzoneManager currentLeader = cluster.getOMLeader(); if (!currentLeader.getOMNodeId().equals(theLeaderOM.getOMNodeId())) { currentLeader.transferLeadership(theLeaderOM.getOMNodeId()); - GenericTestUtils.waitFor(() -> { + BooleanSupplier leadershipCheck = () -> { try { - OzoneManager currentLeaderCheck = cluster.getOMLeader(); - return !currentLeaderCheck.getOMNodeId().equals(currentLeader.getOMNodeId()); + return !cluster.getOMLeader().getOMNodeId().equals(currentLeader.getOMNodeId()); } catch (Exception e) { return false; } - }, 1000, 30000); + }; + GenericTestUtils.waitFor(leadershipCheck, 1000, 30000); } } @@ -140,20 +152,18 @@ public void testOMHAAdminPrivilegesAfterLeadershipChange() throws Exception { addAdminToSpecificOM(currentLeader, TEST_USER); // Verify admin was added - assertTrue(currentLeader.getOmAdminUsernames().contains(TEST_USER), - "Test user should be admin on leader OM"); + assertThat(currentLeader.getOmAdminUsernames()).contains(TEST_USER); // Step 3: Test volume and bucket creation as test user (should succeed) testVolumeAndBucketCreationAsUser(true); // Step 4: Force leadership transfer to another OM node - OzoneManager newLeader = cluster.transferOMLeadershipToAnotherNode(currentLeader); + OzoneManager newLeader = transferLeadershipToAnotherNode(currentLeader); assertNotEquals(leaderNodeId, newLeader.getOMNodeId(), "Leadership should have transferred to a different node"); // Step 5: Verify test user is NOT admin on new leader - assertTrue(!newLeader.getOmAdminUsernames().contains(TEST_USER), - "Test user should NOT be admin on new leader OM"); + assertThat(newLeader.getOmAdminUsernames()).doesNotContain(TEST_USER); // Step 6: Test volume and bucket creation as test user (should fail) testVolumeAndBucketCreationAsUser(false); @@ -336,8 +346,7 @@ public void testKeySetTimesAclEnforcementAfterLeadershipChange() throws Exceptio addAdminToSpecificOM(currentLeader, TEST_USER); // Verify admin was added - assertTrue(currentLeader.getOmAdminUsernames().contains(TEST_USER), - "Test user should be admin on leader OM"); + assertThat(currentLeader.getOmAdminUsernames()).contains(TEST_USER); // Switch to test user and try setTimes as admin (should succeed) UserGroupInformation.setLoginUser(testUserUgi); @@ -359,8 +368,7 @@ public void testKeySetTimesAclEnforcementAfterLeadershipChange() throws Exceptio OzoneManager newLeader = cluster.transferOMLeadershipToAnotherNode(currentLeader); assertNotEquals(leaderNodeId, newLeader.getOMNodeId(), "Leadership should have transferred to a different node"); - assertFalse(newLeader.getOmAdminUsernames().contains(TEST_USER), - "Test user should NOT be admin on new leader OM"); + assertThat(newLeader.getOmAdminUsernames()).doesNotContain(TEST_USER); long anotherMtime = System.currentTimeMillis() + 10000; OMException exception = assertThrows(OMException.class, () -> { @@ -374,6 +382,623 @@ public void testKeySetTimesAclEnforcementAfterLeadershipChange() throws Exceptio } } + /** + * Tests that setQuota ACL check is enforced in preExecute and is leader-specific. + */ + @Test + public void testVolumeSetQuotaAclEnforcementAfterLeadershipChange() throws Exception { + ObjectStore adminObjectStore = client.getObjectStore(); + String testVolume = "quotavol-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + + // Create volume as admin + VolumeArgs volumeArgs = VolumeArgs.newBuilder() + .setOwner(adminUserUgi.getShortUserName()) + .build(); + adminObjectStore.createVolume(testVolume, volumeArgs); + + // Add test user as admin on current leader + OzoneManager currentLeader = cluster.getOMLeader(); + addAdminToSpecificOM(currentLeader, TEST_USER); + assertThat(currentLeader.getOmAdminUsernames()).contains(TEST_USER); + + // Test user should be able to set quota as admin + UserGroupInformation.setLoginUser(testUserUgi); + try (OzoneClient userClient = OzoneClientFactory.getRpcClient(OM_SERVICE_ID, cluster.getConf())) { + ObjectStore userObjectStore = userClient.getObjectStore(); + OzoneVolume userVolume = userObjectStore.getVolume(testVolume); + + OzoneQuota quota1 = OzoneQuota.getOzoneQuota(100L * 1024 * 1024 * 1024, 1000); + userVolume.setQuota(quota1); // Set quota to 100GB + + // Transfer leadership + OzoneManager newLeader = transferLeadershipToAnotherNode(currentLeader); + assertThat(newLeader.getOmAdminUsernames()).doesNotContain(TEST_USER); + + // Should fail on new leader + OzoneQuota quota2 = OzoneQuota.getOzoneQuota(200L * 1024 * 1024 * 1024, 2000); + OMException exception = assertThrows(OMException.class, () -> { + userVolume.setQuota(quota2); + }, "setQuota should fail for non-admin user on new leader"); + assertEquals(PERMISSION_DENIED, exception.getResult()); + } finally { + UserGroupInformation.setLoginUser(adminUserUgi); + } + } + + /** + * Tests that setOwner ACL check is enforced in preExecute and is leader-specific. + */ + @Test + public void testVolumeSetOwnerAclEnforcementAfterLeadershipChange() throws Exception { + ObjectStore adminObjectStore = client.getObjectStore(); + String testVolume = "ownervol-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + + // Create volume as admin + VolumeArgs volumeArgs = VolumeArgs.newBuilder() + .setOwner(adminUserUgi.getShortUserName()) + .build(); + adminObjectStore.createVolume(testVolume, volumeArgs); + + // Add test user as admin on current leader + OzoneManager currentLeader = cluster.getOMLeader(); + addAdminToSpecificOM(currentLeader, TEST_USER); + assertThat(currentLeader.getOmAdminUsernames()).contains(TEST_USER); + + // Test user should be able to change owner as admin + UserGroupInformation.setLoginUser(testUserUgi); + try (OzoneClient userClient = OzoneClientFactory.getRpcClient(OM_SERVICE_ID, cluster.getConf())) { + ObjectStore userObjectStore = userClient.getObjectStore(); + OzoneVolume userVolume = userObjectStore.getVolume(testVolume); + + userVolume.setOwner("newowner"); + + // Transfer leadership + OzoneManager newLeader = transferLeadershipToAnotherNode(currentLeader); + assertThat(newLeader.getOmAdminUsernames()).doesNotContain(TEST_USER); + + // Should fail on new leader + OMException exception = assertThrows(OMException.class, () -> { + userVolume.setOwner("anothernewowner"); + }, "setOwner should fail for non-admin user on new leader"); + assertEquals(PERMISSION_DENIED, exception.getResult()); + } finally { + UserGroupInformation.setLoginUser(adminUserUgi); + } + } + + /** + * Tests that deleteVolume ACL check is enforced in preExecute and is leader-specific. + */ + @Test + public void testVolumeDeleteAclEnforcementAfterLeadershipChange() throws Exception { + ObjectStore adminObjectStore = client.getObjectStore(); + String testVolume1 = "delvol1-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String testVolume2 = "delvol2-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + + // Create volumes as admin + VolumeArgs volumeArgs = VolumeArgs.newBuilder() + .setOwner(adminUserUgi.getShortUserName()) + .build(); + adminObjectStore.createVolume(testVolume1, volumeArgs); + adminObjectStore.createVolume(testVolume2, volumeArgs); + + // Add test user as admin on current leader + OzoneManager currentLeader = cluster.getOMLeader(); + addAdminToSpecificOM(currentLeader, TEST_USER); + assertThat(currentLeader.getOmAdminUsernames()).contains(TEST_USER); + + // Test user should be able to delete volume as admin + UserGroupInformation.setLoginUser(testUserUgi); + try (OzoneClient userClient = OzoneClientFactory.getRpcClient(OM_SERVICE_ID, cluster.getConf())) { + ObjectStore userObjectStore = userClient.getObjectStore(); + + userObjectStore.deleteVolume(testVolume1); + + // Transfer leadership + OzoneManager newLeader = transferLeadershipToAnotherNode(currentLeader); + assertThat(newLeader.getOmAdminUsernames()).doesNotContain(TEST_USER); + + // Should fail on new leader + OMException exception = assertThrows(OMException.class, () -> { + userObjectStore.deleteVolume(testVolume2); + }, "deleteVolume should fail for non-admin user on new leader"); + assertEquals(PERMISSION_DENIED, exception.getResult()); + } finally { + UserGroupInformation.setLoginUser(adminUserUgi); + } + } + + /** + * Tests that setBucketProperty ACL check is enforced in preExecute and is leader-specific. + */ + @Test + public void testBucketSetPropertyAclEnforcementAfterLeadershipChange() throws Exception { + ObjectStore adminObjectStore = client.getObjectStore(); + String testVolume = "bucketpropvol-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String testBucket = "bucketprop-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + + // Create volume and bucket as admin + VolumeArgs volumeArgs = VolumeArgs.newBuilder() + .setOwner(adminUserUgi.getShortUserName()) + .build(); + adminObjectStore.createVolume(testVolume, volumeArgs); + OzoneVolume adminVolume = adminObjectStore.getVolume(testVolume); + + BucketArgs bucketArgs = BucketArgs.newBuilder().build(); + adminVolume.createBucket(testBucket, bucketArgs); + + // Add test user as admin on current leader + OzoneManager currentLeader = cluster.getOMLeader(); + addAdminToSpecificOM(currentLeader, TEST_USER); + assertThat(currentLeader.getOmAdminUsernames()).contains(TEST_USER); + + // Test user should be able to set bucket properties as admin + UserGroupInformation.setLoginUser(testUserUgi); + try (OzoneClient userClient = OzoneClientFactory.getRpcClient(OM_SERVICE_ID, cluster.getConf())) { + ObjectStore userObjectStore = userClient.getObjectStore(); + OzoneVolume userVolume = userObjectStore.getVolume(testVolume); + OzoneBucket userBucket = userVolume.getBucket(testBucket); + + // Set versioning + userBucket.setVersioning(true); + + // Transfer leadership + OzoneManager newLeader = transferLeadershipToAnotherNode(currentLeader); + assertThat(newLeader.getOmAdminUsernames()).doesNotContain(TEST_USER); + + // Should fail on new leader + OMException exception = assertThrows(OMException.class, () -> { + userBucket.setVersioning(false); + }, "setBucketProperty should fail for non-admin user on new leader"); + assertEquals(PERMISSION_DENIED, exception.getResult()); + } finally { + UserGroupInformation.setLoginUser(adminUserUgi); + } + } + + /** + * Tests that setBucketOwner ACL check is enforced in preExecute and is leader-specific. + */ + @Test + public void testBucketSetOwnerAclEnforcementAfterLeadershipChange() throws Exception { + ObjectStore adminObjectStore = client.getObjectStore(); + String testVolume = "bucketownervol-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String testBucket = "bucketowner-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + + // Create volume and bucket as admin + VolumeArgs volumeArgs = VolumeArgs.newBuilder() + .setOwner(adminUserUgi.getShortUserName()) + .build(); + adminObjectStore.createVolume(testVolume, volumeArgs); + OzoneVolume adminVolume = adminObjectStore.getVolume(testVolume); + + BucketArgs bucketArgs = BucketArgs.newBuilder().build(); + adminVolume.createBucket(testBucket, bucketArgs); + + // Add test user as admin on current leader + OzoneManager currentLeader = cluster.getOMLeader(); + addAdminToSpecificOM(currentLeader, TEST_USER); + assertThat(currentLeader.getOmAdminUsernames()).contains(TEST_USER); + + // Test user should be able to set bucket owner as admin + UserGroupInformation.setLoginUser(testUserUgi); + try (OzoneClient userClient = OzoneClientFactory.getRpcClient(OM_SERVICE_ID, cluster.getConf())) { + ObjectStore userObjectStore = userClient.getObjectStore(); + OzoneVolume userVolume = userObjectStore.getVolume(testVolume); + OzoneBucket userBucket = userVolume.getBucket(testBucket); + + // Set new owner + userBucket.setOwner("newowner"); + + // Transfer leadership + OzoneManager newLeader = transferLeadershipToAnotherNode(currentLeader); + assertThat(newLeader.getOmAdminUsernames()).doesNotContain(TEST_USER); + + // Should fail on new leader + OMException exception = assertThrows(OMException.class, () -> { + userBucket.setOwner("anothernewowner"); + }, "setBucketOwner should fail for non-admin user on new leader"); + assertEquals(PERMISSION_DENIED, exception.getResult()); + } finally { + UserGroupInformation.setLoginUser(adminUserUgi); + } + } + + /** + * Tests that deleteBucket ACL check is enforced in preExecute and is leader-specific. + */ + @Test + public void testBucketDeleteAclEnforcementAfterLeadershipChange() throws Exception { + ObjectStore adminObjectStore = client.getObjectStore(); + String testVolume = "bucketdelvol-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String testBucket1 = "bucketdel1-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String testBucket2 = "bucketdel2-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + + // Create volume and buckets as admin + VolumeArgs volumeArgs = VolumeArgs.newBuilder() + .setOwner(adminUserUgi.getShortUserName()) + .build(); + adminObjectStore.createVolume(testVolume, volumeArgs); + OzoneVolume adminVolume = adminObjectStore.getVolume(testVolume); + + BucketArgs bucketArgs = BucketArgs.newBuilder().build(); + adminVolume.createBucket(testBucket1, bucketArgs); + adminVolume.createBucket(testBucket2, bucketArgs); + + // Add test user as admin on current leader + OzoneManager currentLeader = cluster.getOMLeader(); + addAdminToSpecificOM(currentLeader, TEST_USER); + assertThat(currentLeader.getOmAdminUsernames()).contains(TEST_USER); + + // Test user should be able to delete bucket as admin + UserGroupInformation.setLoginUser(testUserUgi); + try (OzoneClient userClient = OzoneClientFactory.getRpcClient(OM_SERVICE_ID, cluster.getConf())) { + ObjectStore userObjectStore = userClient.getObjectStore(); + OzoneVolume userVolume = userObjectStore.getVolume(testVolume); + + userVolume.deleteBucket(testBucket1); + + // Transfer leadership + OzoneManager newLeader = transferLeadershipToAnotherNode(currentLeader); + assertThat(newLeader.getOmAdminUsernames()).doesNotContain(TEST_USER); + + // Should fail on new leader + OMException exception = assertThrows(OMException.class, () -> { + userVolume.deleteBucket(testBucket2); + }, "deleteBucket should fail for non-admin user on new leader"); + assertEquals(PERMISSION_DENIED, exception.getResult()); + } finally { + UserGroupInformation.setLoginUser(adminUserUgi); + } + } + + /** + * Tests that deleteKeys (bulk) ACL check is enforced in preExecute and is leader-specific. + * + *

This test verifies that when using the bulk deleteKeys API: + *

+ * + *

The test flow: + *

    + *
  1. Create test volume, bucket, and multiple keys as admin
    2. + *
  2. Add test user as admin on current leader
    3. + *
  3. Test user successfully deletes first key using bulk API
    4. + *
  4. Transfer leadership to node where test user is NOT admin
    5. + *
  5. Test user attempts to delete second key - PARTIAL_DELETE exception is thrown
    6. + *
  6. Verify the key still exists (was not deleted due to ACL check failure)
    7. + *
+ */ + @Test + public void testKeysDeleteAclEnforcementAfterLeadershipChange() throws Exception { + ObjectStore adminObjectStore = client.getObjectStore(); + String testVolume = "keysdelvol-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String testBucket = "keysdel-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String keyName1 = "key1-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String keyName2 = "key2-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + + // Step 1: Create volume, bucket, and keys as admin + VolumeArgs volumeArgs = VolumeArgs.newBuilder() + .setOwner(adminUserUgi.getShortUserName()) + .build(); + adminObjectStore.createVolume(testVolume, volumeArgs); + OzoneVolume adminVolume = adminObjectStore.getVolume(testVolume); + + BucketArgs bucketArgs = BucketArgs.newBuilder().build(); + adminVolume.createBucket(testBucket, bucketArgs); + OzoneBucket adminBucket = adminVolume.getBucket(testBucket); + + // Create test keys + try (OzoneOutputStream out = adminBucket.createKey(keyName1, 0)) { + out.write("test data 1".getBytes(UTF_8)); + } + try (OzoneOutputStream out = adminBucket.createKey(keyName2, 0)) { + out.write("test data 2".getBytes(UTF_8)); + } + + // Step 2: Add test user as admin on current leader + OzoneManager currentLeader = cluster.getOMLeader(); + String originalLeaderNodeId = currentLeader.getOMNodeId(); + addAdminToSpecificOM(currentLeader, TEST_USER); + assertThat(currentLeader.getOmAdminUsernames()).contains(TEST_USER); + + // Step 3: Test user deletes first key successfully using bulk API + UserGroupInformation.setLoginUser(testUserUgi); + try (OzoneClient userClient = OzoneClientFactory.getRpcClient(OM_SERVICE_ID, cluster.getConf())) { + ObjectStore userObjectStore = userClient.getObjectStore(); + OzoneVolume userVolume = userObjectStore.getVolume(testVolume); + OzoneBucket userBucket = userVolume.getBucket(testBucket); + + // Use bulk deleteKeys API (this supports ACL filtering) + userBucket.deleteKeys(Collections.singletonList(keyName1)); + + // Verify key1 was deleted + OMException ex1 = assertThrows(OMException.class, () -> userBucket.getKey(keyName1), + "Key1 should be deleted"); + assertEquals(OMException.ResultCodes.KEY_NOT_FOUND, ex1.getResult()); + + // Step 4: Transfer leadership to another node where test user is NOT admin + OzoneManager newLeader = transferLeadershipToAnotherNode(currentLeader); + assertNotEquals(originalLeaderNodeId, newLeader.getOMNodeId(), + "Leadership should have transferred to a different node"); + assertThat(newLeader.getOmAdminUsernames()).doesNotContain(TEST_USER); + + // Step 5: Attempt to delete second key - ACL check fails in preExecute. + // The server filters out the denied key and returns PARTIAL_DELETE status, + // which the non-quiet deleteKeys client translates to an OMException. + OMException deleteException = assertThrows(OMException.class, () -> + userBucket.deleteKeys(Collections.singletonList(keyName2)), + "deleteKeys should fail with PARTIAL_DELETE when ACL check fails"); + assertEquals(PARTIAL_DELETE, deleteException.getResult()); + } finally { + UserGroupInformation.setLoginUser(adminUserUgi); + } + + // Step 6: Verify key2 still exists (was filtered out due to ACL failure) + OzoneKey key2 = adminBucket.getKey(keyName2); + assertNotNull(key2, "Key2 should still exist after being filtered out by ACL check"); + assertEquals(keyName2, key2.getName()); + } + + /** + * Tests that renameKeys (bulk) ACL check is enforced in preExecute and is leader-specific. + * + *

This test verifies that when using the bulk renameKeys API: + *

+ * + *

The test flow: + *

    + *
  1. Create test volume, bucket, and multiple keys as admin with LEGACY bucket layout
    2. + *
  2. Add test user as admin on current leader
    3. + *
  3. Test user successfully renames first key using bulk API
    4. + *
  4. Transfer leadership to node where test user is NOT admin
    5. + *
  5. Test user attempts to rename second key - PARTIAL_RENAME exception is thrown
    6. + *
  6. Verify the key still has original name (was not renamed due to ACL check failure)
    7. + *
+ * + *

Note: This test uses LEGACY bucket layout because the bulk renameKeys API is deprecated + * and not supported for FILE_SYSTEM_OPTIMIZED layouts. + */ + @Test + public void testKeysRenameAclEnforcementAfterLeadershipChange() throws Exception { + ObjectStore adminObjectStore = client.getObjectStore(); + String testVolume = "keysrenamevol-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String testBucket = "keysrename-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String keyName1 = "key1-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String keyName2 = "key2-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String newKeyName1 = "newkey1-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String newKeyName2 = "newkey2-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + + // Step 1: Create volume, bucket with LEGACY layout, and keys as admin + VolumeArgs volumeArgs = VolumeArgs.newBuilder() + .setOwner(adminUserUgi.getShortUserName()) + .build(); + adminObjectStore.createVolume(testVolume, volumeArgs); + OzoneVolume adminVolume = adminObjectStore.getVolume(testVolume); + + // Use LEGACY bucket layout since bulk renameKeys is not supported for FSO + BucketArgs bucketArgs = BucketArgs.newBuilder() + .setBucketLayout(BucketLayout.LEGACY) + .build(); + adminVolume.createBucket(testBucket, bucketArgs); + OzoneBucket adminBucket = adminVolume.getBucket(testBucket); + + // Create test keys + try (OzoneOutputStream out = adminBucket.createKey(keyName1, 0)) { + out.write("test data 1".getBytes(UTF_8)); + } + try (OzoneOutputStream out = adminBucket.createKey(keyName2, 0)) { + out.write("test data 2".getBytes(UTF_8)); + } + + // Step 2: Add test user as admin on current leader + OzoneManager currentLeader = cluster.getOMLeader(); + String originalLeaderNodeId = currentLeader.getOMNodeId(); + addAdminToSpecificOM(currentLeader, TEST_USER); + assertThat(currentLeader.getOmAdminUsernames()).contains(TEST_USER); + + // Step 3: Test user renames first key successfully using bulk API + UserGroupInformation.setLoginUser(testUserUgi); + try (OzoneClient userClient = OzoneClientFactory.getRpcClient(OM_SERVICE_ID, cluster.getConf())) { + ObjectStore userObjectStore = userClient.getObjectStore(); + OzoneVolume userVolume = userObjectStore.getVolume(testVolume); + OzoneBucket userBucket = userVolume.getBucket(testBucket); + + // Use bulk renameKeys API (this supports ACL filtering) + Map renameMap1 = new HashMap<>(); + renameMap1.put(keyName1, newKeyName1); + userBucket.renameKeys(renameMap1); + + // Verify key1 was renamed + OMException ex1 = assertThrows(OMException.class, () -> userBucket.getKey(keyName1), + "Original key1 should not exist after rename"); + assertEquals(OMException.ResultCodes.KEY_NOT_FOUND, ex1.getResult()); + assertNotNull(userBucket.getKey(newKeyName1), "Renamed key1 should exist"); + + // Step 4: Transfer leadership to another node where test user is NOT admin + OzoneManager newLeader = transferLeadershipToAnotherNode(currentLeader); + assertNotEquals(originalLeaderNodeId, newLeader.getOMNodeId(), + "Leadership should have transferred to a different node"); + assertThat(newLeader.getOmAdminUsernames()).doesNotContain(TEST_USER); + + // Step 5: Attempt to rename second key - ACL check fails in preExecute. + // The server filters out the denied key pair and returns PARTIAL_RENAME status, + // which the renameKeys client (no quiet mode) translates to an OMException. + Map renameMap2 = new HashMap<>(); + renameMap2.put(keyName2, newKeyName2); + OMException renameException = assertThrows(OMException.class, () -> + userBucket.renameKeys(renameMap2), + "renameKeys should fail with PARTIAL_RENAME when ACL check fails"); + assertEquals(PARTIAL_RENAME, renameException.getResult()); + } finally { + UserGroupInformation.setLoginUser(adminUserUgi); + } + + // Step 6: Verify key2 still has original name (was filtered out due to ACL failure) + OzoneKey key2 = adminBucket.getKey(keyName2); + assertNotNull(key2, "Original key2 should still exist after being filtered out by ACL check"); + assertEquals(keyName2, key2.getName()); + + // Verify new key name doesn't exist + OMException ex2 = assertThrows(OMException.class, () -> adminBucket.getKey(newKeyName2), + "New key name should not exist after ACL filtering"); + assertEquals(OMException.ResultCodes.KEY_NOT_FOUND, ex2.getResult()); + } + + /** + * Tests that key ACL operations (addAcl, removeAcl, setAcl) are enforced in preExecute + * and are leader-specific. Uses FILE_SYSTEM_OPTIMIZED bucket layout. + * + *

This test verifies that ACL operations on keys work correctly across leadership changes: + *

+ */ + @Test + public void testKeyAclOperationsEnforcementAfterLeadershipChangeWithFSO() throws Exception { + ObjectStore adminObjectStore = client.getObjectStore(); + String testVolume = "keyaclvol-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String testBucket = "keyaclbucket-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + String keyName = "keyacl-" + + RandomStringUtils.secure().nextAlphabetic(5).toLowerCase(Locale.ROOT); + + // Step 1: Create volume, bucket with FSO layout, and key as admin + VolumeArgs volumeArgs = VolumeArgs.newBuilder() + .setOwner(adminUserUgi.getShortUserName()) + .build(); + adminObjectStore.createVolume(testVolume, volumeArgs); + OzoneVolume adminVolume = adminObjectStore.getVolume(testVolume); + + // Use FILE_SYSTEM_OPTIMIZED bucket layout + BucketArgs bucketArgs = BucketArgs.newBuilder() + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .build(); + adminVolume.createBucket(testBucket, bucketArgs); + OzoneBucket adminBucket = adminVolume.getBucket(testBucket); + + // Create a key as admin (so test user is NOT the owner) + try (OzoneOutputStream out = adminBucket.createKey(keyName, 0)) { + out.write("test data for ACL operations".getBytes(UTF_8)); + } + + OzoneKey key = adminBucket.getKey(keyName); + assertNotNull(key, "Key should be created successfully"); + + // Create OzoneObj for ACL operations + OzoneObj keyObj = OzoneObjInfo.Builder.newBuilder() + .setVolumeName(testVolume) + .setBucketName(testBucket) + .setKeyName(keyName) + .setResType(OzoneObj.ResourceType.KEY) + .setStoreType(OzoneObj.StoreType.OZONE) + .build(); + + int originalAclCount = adminObjectStore.getAcl(keyObj).size(); + + // Step 2: Add test user as admin on current leader + OzoneManager currentLeader = cluster.getOMLeader(); + String leaderNodeId = currentLeader.getOMNodeId(); + addAdminToSpecificOM(currentLeader, TEST_USER); + assertThat(currentLeader.getOmAdminUsernames()).contains(TEST_USER); + + // Step 3: Test user performs ACL operations as admin (should succeed) + UserGroupInformation.setLoginUser(testUserUgi); + try (OzoneClient userClient = OzoneClientFactory.getRpcClient(OM_SERVICE_ID, cluster.getConf())) { + ObjectStore userObjectStore = userClient.getObjectStore(); + + // Add ACL - should succeed + OzoneAcl addAcl = OzoneAcl.parseAcl("user:anotheruser:rw[ACCESS]"); + boolean addResult = userObjectStore.addAcl(keyObj, addAcl); + assertThat(addResult).isTrue(); + + // Verify ACL was added + List acls = userObjectStore.getAcl(keyObj); + assertThat(acls).hasSize(originalAclCount + 1); + assertThat(acls).contains(addAcl); + + // Set ACL - should succeed + OzoneAcl setAcl = OzoneAcl.parseAcl("user:setuser:rwx[ACCESS]"); + boolean setResult = userObjectStore.setAcl(keyObj, Collections.singletonList(setAcl)); + assertThat(setResult).isTrue(); + + // Verify ACL was set (replaced all previous ACLs) + acls = userObjectStore.getAcl(keyObj); + assertThat(acls).hasSize(1); + assertThat(acls).contains(setAcl); + + // Step 4: Transfer leadership to another node where test user is NOT admin + OzoneManager newLeader = transferLeadershipToAnotherNode(currentLeader); + assertNotEquals(leaderNodeId, newLeader.getOMNodeId(), + "Leadership should have transferred to a different node"); + assertThat(newLeader.getOmAdminUsernames()).doesNotContain(TEST_USER); + + // Step 5: Try ACL operations on new leader - should fail with PERMISSION_DENIED + OzoneAcl anotherAcl = OzoneAcl.parseAcl("user:yetanotheruser:r[ACCESS]"); + + // Add ACL should fail + OMException addException = assertThrows(OMException.class, () -> { + userObjectStore.addAcl(keyObj, anotherAcl); + }, "addAcl should fail for non-admin user on new leader"); + assertEquals(PERMISSION_DENIED, addException.getResult(), + "Should get PERMISSION_DENIED when ACL check fails in preExecute"); + + // Remove ACL should fail + OMException removeException = assertThrows(OMException.class, () -> { + userObjectStore.removeAcl(keyObj, setAcl); + }, "removeAcl should fail for non-admin user on new leader"); + assertEquals(PERMISSION_DENIED, removeException.getResult(), + "Should get PERMISSION_DENIED when ACL check fails in preExecute"); + + // Set ACL should fail + OzoneAcl newSetAcl = OzoneAcl.parseAcl("user:failuser:w[ACCESS]"); + OMException setException = assertThrows(OMException.class, () -> { + userObjectStore.setAcl(keyObj, Collections.singletonList(newSetAcl)); + }, "setAcl should fail for non-admin user on new leader"); + assertEquals(PERMISSION_DENIED, setException.getResult(), + "Should get PERMISSION_DENIED when ACL check fails in preExecute"); + + } finally { + UserGroupInformation.setLoginUser(adminUserUgi); + } + + // Step 6: Verify the ACLs remain unchanged (operations were rejected in preExecute) + List finalAcls = adminObjectStore.getAcl(keyObj); + assertThat(finalAcls).hasSize(1); + assertThat(finalAcls).contains(OzoneAcl.parseAcl("user:setuser:rwx[ACCESS]")); + } + /** * Helper method to check if volume exists. */ @@ -385,4 +1010,44 @@ private boolean volumeExists(ObjectStore store, String volumeName) { return false; } } + + /** + * Transfers leadership from current leader to another OM node. + * + * @param currentLeader the current leader OM + * @return the new leader OM after transfer + */ + private OzoneManager transferLeadershipToAnotherNode(OzoneManager currentLeader) throws Exception { + // Get list of all OMs + List omList = new ArrayList<>(cluster.getOzoneManagersList()); + + // Remove current leader from list + omList.remove(currentLeader); + + // Select the first alternative OM as target + OzoneManager targetOM = omList.get(0); + String targetNodeId = targetOM.getOMNodeId(); + + // Transfer leadership + currentLeader.transferLeadership(targetNodeId); + + // Wait for leadership transfer to complete + BooleanSupplier leadershipTransferCheck = () -> { + try { + return !cluster.getOMLeader().getOMNodeId().equals(currentLeader.getOMNodeId()); + } catch (Exception e) { + return false; + } + }; + GenericTestUtils.waitFor(leadershipTransferCheck, 1000, 30000); + + // Verify leadership change + cluster.waitForLeaderOM(); + OzoneManager newLeader = cluster.getOMLeader(); + + assertEquals(targetNodeId, newLeader.getOMNodeId(), + "Leadership should have transferred to target OM"); + + return newLeader; + } }