HDDS-15404. Enable short-circuit unit test in CI

.github/workflows/ci.yml

@@ -340,7 +340,7 @@ jobs:
       pre-script: sudo hostname localhost
       ratis-args: ${{ inputs.ratis_args }}
       script: integration
-      script-args: -Ptest-${{ matrix.profile }} -Drocks_tools_native
+      script-args: -Ptest-${{ matrix.profile }} -Phadoop-native-lib -Drocks_tools_native
       sha: ${{ needs.build-info.outputs.sha }}
       split: ${{ matrix.profile }}
       timeout-minutes: 90

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java

@@ -21,9 +21,9 @@
 import java.io.IOException;
 import java.net.SocketTimeoutException;
 import java.nio.channels.AsynchronousCloseException;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -69,8 +69,8 @@ public final class XceiverServerDomainSocket implements XceiverServerSpi, Runnab
   private DomainSocket domainSocket;
   private final ConfigurationSource config;
   private final String threadPrefix;
-  private final HashMap<DomainPeer, Thread> peers = new HashMap<>();
-  private final HashMap<DomainPeer, Receiver> peersReceiver = new HashMap<>();
+  private final ConcurrentHashMap<DomainPeer, Thread> peers = new ConcurrentHashMap<>();
+  private final ConcurrentHashMap<DomainPeer, Receiver> peersReceiver = new ConcurrentHashMap<>();
   private int readTimeoutMs;
   private int writeTimeoutMs;
   private final ThreadPoolExecutor readExecutors;
@@ -162,8 +162,7 @@ public void stop() {
       if (server != null) {
         try {
           if (domainSocket != null) {
-            // TODO: once HADOOP-19261 is merged, change it to domainSocket.close(true);
-            domainSocket.close();
+            domainSocket.close(true);
             LOG.info("UNIX domain socket server listening on {} is stopped", domainSocket.getPath());
           }
         } catch (IOException e) {

hadoop-ozone/integration-test/pom.xml

@@ -571,6 +571,17 @@
           <proc>none</proc>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <argLine>${maven-surefire-plugin.argLine} ${maven-surefire-plugin.argLineAccessArgs} @{argLine} -Djava.library.path=${project.basedir}/../../target/native-lib</argLine>
+          <environmentVariables>
+            <LD_LIBRARY_PATH>${project.basedir}/../../target/native-lib</LD_LIBRARY_PATH>
+            <DYLD_LIBRARY_PATH>${project.basedir}/../../target/native-lib</DYLD_LIBRARY_PATH>
+          </environmentVariables>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverServerDomainSocket.java → hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/TestXceiverServerDomainSocket.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hdds.scm;
+package org.apache.hadoop.hdds;
 
 import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
 import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type.GetBlock;
@@ -59,6 +59,8 @@
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.pipeline.MockPipeline;
 import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.scm.storage.DomainSocketFactory;
@@ -86,7 +88,6 @@
 import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
 import org.apache.hadoop.utils.FaultInjectorImpl;
 import org.apache.ozone.test.GenericTestUtils;
-import org.apache.ozone.test.tag.Unhealthy;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
@@ -96,6 +97,11 @@
 
 /**
  * Tests the XceiverServerDomainSocket class.
+ * Add Environment variables
+ *   LD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib
+ *   DYLD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib
+ *   to intellij run configuration to run it locally.
+ *   Dynamically set the java.library.path in java code doesn't affect the library loading
  */
 @Timeout(300)
 public class TestXceiverServerDomainSocket {
@@ -128,7 +134,6 @@ public static void setup() {
   }
 
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   public void testIllegalDomainPathConfiguration() {
     // empty domain path
     conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, "");
@@ -172,7 +177,6 @@ public void testIllegalDomainPathConfiguration() {
   }
 
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   public void testExistingDomainPath() {
     // an existing domain path, the existing file is override and changed from a normal file to a socket file
     conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath());
@@ -190,7 +194,6 @@ public void testExistingDomainPath() {
   }
 
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   public void testDomainPathPermission() {
     // write from everyone is not allowed (permission too open)
     assertTrue(dir.setWritable(true, false));
@@ -287,7 +290,6 @@ public void testDomainPathPermission() {
    * On Linux, when there is still open file handle of a deleted file, the file handle remains open and can still
    * be used to read and write the file.
    */
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   @ParameterizedTest
   @CsvSource({
       "true, true",
@@ -371,7 +373,6 @@ public void testReadWrite(boolean deleteFileBeforeRead, boolean deleteFileDuring
    * Test concurrent read/write.
    */
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   public void testConcurrentReadWrite() throws IOException {
     conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath());
     ContainerMetrics containerMetrics = ContainerMetrics.create(conf);
@@ -467,7 +468,6 @@ public void testConcurrentReadWrite() throws IOException {
    * Test server is not listening.
    */
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   public void testServerNotListening() {
     conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath());
     DomainSocketFactory factory = DomainSocketFactory.getInstance(conf);
@@ -488,7 +488,6 @@ public void testServerNotListening() {
    * Although socket can be created, read will fail, write can succeed.
    */
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   public void testServerNotStart() {
     conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath());
     DomainSocketFactory factory = DomainSocketFactory.getInstance(conf);
@@ -522,7 +521,6 @@ public void testServerNotStart() {
   }
 
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   public void testReadTimeout() throws InterruptedException {
     conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath());
     conf.set(OzoneConfigKeys.OZONE_CLIENT_READ_TIMEOUT, "2s");
@@ -558,7 +556,6 @@ public void testReadTimeout() throws InterruptedException {
    * read timeout happens.
    */
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   public void testReceiverDaemonStartSlow() {
     conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath());
     DomainSocketFactory factory = DomainSocketFactory.getInstance(conf);
@@ -586,7 +583,6 @@ public void testReceiverDaemonStartSlow() {
   }
 
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   public void testMaxXceiverCount() throws IOException, InterruptedException {
     conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath());
     DatanodeConfiguration datanodeConfiguration = conf.getObject(DatanodeConfiguration.class);
@@ -650,7 +646,6 @@ public void testMaxXceiverCount() throws IOException, InterruptedException {
    * will treat it as a critical error, close the connection.
    */
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   public void testSendIrrelevantMessage() {
     conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath());
     DomainSocketFactory factory = DomainSocketFactory.getInstance(conf);
@@ -677,7 +672,6 @@ public void testSendIrrelevantMessage() {
   }
 
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   public void testSendUnsupportedRequest() throws IOException {
     conf.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath());
     DomainSocketFactory factory = DomainSocketFactory.getInstance(conf);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientManagerSC.java

@@ -31,7 +31,6 @@
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.container.common.SCMTestUtils;
-import org.apache.ozone.test.tag.Unhealthy;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -40,6 +39,11 @@
 
 /**
  * Test for short-circuit enabled XceiverClientManager.
+ * Add Environment variables
+ *   LD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib
+ *   DYLD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib
+ *   to intellij run configuration to run it locally.
+ *   Dynamically set the java.library.path in java code doesn't affect the library loading
  */
 @Timeout(300)
 public class TestXceiverClientManagerSC {
@@ -75,7 +79,6 @@ public static void shutdown() {
   }
 
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   public void testAllocateShortCircuitClient() throws IOException {
     try (XceiverClientManager clientManager = new XceiverClientManager(config,
         config.getObject(ScmClientConfig.class), null)) {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestDomainSocketFactory.java

@@ -25,12 +25,16 @@
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.scm.OzoneClientConfig;
 import org.apache.hadoop.hdds.scm.storage.DomainSocketFactory;
-import org.apache.ozone.test.tag.Unhealthy;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
 /**
  * Tests for {@link DomainSocketFactory}'s functionality.
+ * For local intellij run, please follow the steps below:
+ * Add Environment variables
+ *  LD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib
+ *  DYLD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib
+ *  to intellij run configuration.
  */
 public class TestDomainSocketFactory {
 
@@ -56,7 +60,6 @@ private DomainSocketFactory getDomainSocketFactory() {
   }
 
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   public void testShortCircuitDisableTemporary() {
     DomainSocketFactory factory = getDomainSocketFactory();
     try {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestShortCircuitChunkInputStream.java → hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestLocalChunkInputStream.java

@@ -44,17 +44,21 @@
 import org.apache.hadoop.ozone.container.keyvalue.ContainerLayoutTestInfo;
 import org.apache.hadoop.ozone.om.TestBucket;
 import org.apache.ozone.test.GenericTestUtils;
-import org.apache.ozone.test.tag.Unhealthy;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.io.TempDir;
 import org.slf4j.event.Level;
 
 /**
  * Tests {@link LocalChunkInputStream}.
+ * For local intellij run, please follow the steps below:
+ *  Add Environment variables
+ *  LD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib
+ *  DYLD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib
+ *  to intellij run configuration.
  */
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
-public class TestShortCircuitChunkInputStream extends TestChunkInputStream {
+public class TestLocalChunkInputStream extends TestChunkInputStream {
 
   @TempDir
   private File dir;
@@ -89,7 +93,6 @@ ReplicationConfig getRepConfig() {
    */
   @ContainerLayoutTestInfo.ContainerTest
   @Override
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   void testAll(ContainerLayoutVersion layout) throws Exception {
     try (OzoneClient client = getCluster().newClient()) {
       updateConfig(layout);
@@ -115,7 +118,6 @@ void testAll(ContainerLayoutVersion layout) throws Exception {
   }
 
   @Test
-  @Unhealthy("Run it locally since it requires libhadoop.so.")
   void testFallbackToGrpc() throws Exception {
     try (OzoneClient client = getCluster().newClient()) {
       assumeTrue(DomainSocketFactory.getInstance(getCluster().getConf()).isServiceReady());