[SYSTEMDS-3876] Include MKL and OpenBLAS in testing docker image

.gitignore

@@ -143,6 +143,7 @@ scripts/perftest/fed/temp
 src/test/scripts/functions/iogen/*.raw
 src/test/scripts/functions/pipelines/intermediates/regression/*
 src/test/scripts/functions/pipelines/intermediates/classification/*
+src/test/scripts/functions/dnn/nativeBLAS/*
 
 venv
 venv/*
@@ -156,4 +157,4 @@ docker/mountFolder/*.bin
 docker/mountFolder/*.bin.mtd
 
 SEAL-*/
-
+OpenBLAS-*/

docker/testsysds.Dockerfile

@@ -18,8 +18,8 @@
 # under the License.
 #
 #-------------------------------------------------------------
-# Stage 1: Build SEAL
-FROM ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 AS seal-build
+# Stage 1: Build SEAL, OpenBLAS, MKL
+FROM ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 AS build
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
@@ -28,18 +28,52 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     tar \
     git \
     ca-certificates \
+    gnupg \
+    python3 \
+    python3-pip \
+    python3-venv \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /seal
 
 # Install SEAL
-RUN wget -qO- https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz | tar xzf - \
-    && cd SEAL-3.7.0 \
+ARG SEAL_VERSION="3.7.0"
+RUN wget -qO- https://github.com/microsoft/SEAL/archive/refs/tags/v${SEAL_VERSION}.tar.gz | tar xzf - \
+    && cd SEAL-${SEAL_VERSION} \
     && cmake -S . -B build -DBUILD_SHARED_LIBS=ON \
     && cmake --build build \
     && cmake --install build --prefix /seal-install
 
-# Stage 2: Final image with R, JDK, Maven, SEAL
+WORKDIR /openBLAS
+
+# Install OpenBLAS
+ARG OPENBLAS_VERSION="0.3.26"
+RUN wget -qO- https://github.com/OpenMathLib/OpenBLAS/archive/refs/tags/v${OPENBLAS_VERSION}.tar.gz | tar xzf - \
+    && cd OpenBLAS-${OPENBLAS_VERSION} \
+    && make -j$(nproc) \
+    && make install PREFIX=/openBLAS-install
+
+WORKDIR /mkl
+
+# Install old MKL, since SystemDS depends on intel MKL <= 2019.x
+# Only package distro which has such old version available is PyPi
+ENV PATH="/mkl-install/bin:$PATH"
+RUN python3 -m venv /mkl-install && \
+    pip install mkl-devel==2018.0.0
+
+# Delete unused libraries, since MKL libraries take up a lot of disk space
+RUN find /mkl-install/lib \( \ 
+    -name '*ilp64*' -o \
+    -name 'libmkl_gnu_thread*' -o \
+    -name 'libmkl_tbb_thread*' -o \
+    -name '*_openmpi_*' -o \
+    -name '*_intelmpi_*' -o \
+    -name 'libmkl_vml*' -o \
+    -name 'libmkl_scalapack*' \
+    \) -delete
+
+
+# Stage 2: Final image with R, JDK, Maven, SEAL, OpenBLAS, MKL
 FROM ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 
 
 WORKDIR /usr/src/
@@ -71,6 +105,7 @@ RUN apt-get install -y --no-install-recommends \
     libssl-dev \
 	r-base-dev \
 	r-base-core \
+    gfortran \
     && apt-get clean && rm -rf /var/lib/apt/lists/* \
 	&& mkdir -p /usr/lib/jvm \
 	&& wget -qO- \
@@ -101,8 +136,16 @@ RUN mkdir -p $HADOOP_HOME/lib/native \
 	rm -rf native
 
 # Copy SEAL
-COPY --from=seal-build /seal-install/lib/ /usr/local/lib/
-COPY --from=seal-build /seal-install/include/ /usr/local/include/
+COPY --from=build /seal-install/lib/ /usr/local/lib/
+COPY --from=build /seal-install/include/ /usr/local/include/
+
+# Copy OpenBLAS
+COPY --from=build /openBLAS-install/lib/ /usr/local/lib/
+COPY --from=build /openBLAS-install/include/ /usr/local/include/
+
+# Copy MKL
+COPY --from=build /mkl-install/include /usr/local/include/
+COPY --from=build /mkl-install/lib /usr/local/lib/
 
 ENV LD_LIBRARY_PATH=/opt/hadoop/lib/native;/usr/local/lib/
 

src/test/config/SystemDS-config-MKL.xml

@@ -0,0 +1,23 @@
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+-->
+
+<root>
+	<sysds.native.blas>mkl</sysds.native.blas>
+	<sysds.native.blas.directory>/usr/local/lib</sysds.native.blas.directory>
+</root>

src/test/config/SystemDS-config-OpenBLAS.xml

@@ -0,0 +1,23 @@
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+-->
+
+<root>
+	<sysds.native.blas>openblas</sysds.native.blas>
+	<sysds.native.blas.directory>/usr/local/lib</sysds.native.blas.directory>
+</root>

src/test/java/org/apache/sysds/test/component/matrix/MatrixMultiplyTest.java

@@ -28,19 +28,17 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.sysds.runtime.functionobjects.Multiply;
-import org.apache.sysds.runtime.functionobjects.Plus;
 import org.apache.sysds.runtime.matrix.data.MatrixBlock;
-import org.apache.sysds.runtime.matrix.operators.AggregateBinaryOperator;
-import org.apache.sysds.runtime.matrix.operators.AggregateOperator;
+import org.apache.sysds.runtime.matrix.data.LibMatrixNative;
 import org.apache.sysds.test.TestUtils;
+import org.apache.sysds.test.AutomatedTestBase;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
 @RunWith(value = Parameterized.class)
-public class MatrixMultiplyTest {
+public class MatrixMultiplyTest extends AutomatedTestBase{
 	protected static final Log LOG = LogFactory.getLog(MatrixMultiplyTest.class.getName());
 
 	// left side
@@ -278,9 +276,16 @@ private static String size(MatrixBlock a) {
 	}
 
 	private static MatrixBlock multiply(MatrixBlock a, MatrixBlock b, int k) {
-		AggregateOperator agg = new AggregateOperator(0, Plus.getPlusFnObject());
-		AggregateBinaryOperator mult = new AggregateBinaryOperator(Multiply.getMultiplyFnObject(), agg, k);
-		return a.aggregateBinaryOperations(a, b, mult);
+		MatrixBlock ret = new MatrixBlock();
+		try {
+			LibMatrixNative.matrixMult(a,b,ret,k);
+		} catch (Exception e) {
+			e.printStackTrace();
+			throw e;
+		}
+		return ret;
 	}
 
+	@Override
+	public void setUp(){}
 }

src/test/java/org/apache/sysds/test/component/matrix/nativeBLAS/MatrixMultiplyMKLTest.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sysds.test.component.matrix.nativeBLAS;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.lang.Math;
+import java.lang.String;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.conf.DMLConfig;
+import org.apache.sysds.conf.ConfigurationManager;
+import org.apache.sysds.utils.NativeHelper;
+import org.apache.sysds.test.TestConfiguration;
+import org.apache.sysds.test.TestUtils;
+import org.apache.sysds.test.component.matrix.MatrixMultiplyTest;
+
+public class MatrixMultiplyMKLTest extends MatrixMultiplyTest{
+	protected static final Log LOG = LogFactory.getLog(MatrixMultiplyTest.class.getName());
+
+	private final static String TEST_DIR = "component/matrix/nativeBLAS/";
+
+	protected String getTestClassDir() {
+		return getTestDir() + this.getClass().getSimpleName() + "/";
+	}
+
+	protected String getTestName() {
+		return "matrixMultiplyMKL";
+	}
+
+	protected String getTestDir() {
+		return TEST_DIR;
+	}
+
+    public MatrixMultiplyMKLTest(int i, int j, int k, double s, double s2, int p, boolean self) {
+		super(i,j,k,s,s2,p,self);
+	}
+
+	// TODO: test not working with native BLAS
+	@Override
+	public void testLeftNonContiguous(){}
+
+	@Override
+	protected File getConfigTemplateFile() {
+		return new File("./src/test/config/SystemDS-config-MKL.xml");
+	}
+
+	@Override
+	public void setUp() {
+		try {
+			String testname = getTestName() + String.valueOf(Math.random()*5);
+			addTestConfiguration(testname, new TestConfiguration(getTestClassDir(), testname));
+			loadTestConfiguration(getTestConfiguration(testname));
+
+			DMLConfig conf = new DMLConfig(getCurConfigFile().getPath());
+			ConfigurationManager.setLocalConfig(conf);
+
+			assertEquals(true, NativeHelper.isNativeLibraryLoaded());
+		} catch(Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
+
+	@Override
+	public void tearDown() {
+		TestUtils.clearDirectory(getCurLocalTempDir().getPath());
+		TestUtils.removeDirectories(new String[]{getCurLocalTempDir().getPath()});
+	}
+}

src/test/java/org/apache/sysds/test/component/matrix/nativeBLAS/MatrixMultiplyOpenBLASTest.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sysds.test.component.matrix.nativeBLAS;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.lang.Math;
+import java.lang.String;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.conf.DMLConfig;
+import org.apache.sysds.conf.ConfigurationManager;
+import org.apache.sysds.utils.NativeHelper;
+import org.apache.sysds.test.TestConfiguration;
+import org.apache.sysds.test.TestUtils;
+import org.apache.sysds.test.component.matrix.MatrixMultiplyTest;
+
+public class MatrixMultiplyOpenBLASTest extends MatrixMultiplyTest{
+	protected static final Log LOG = LogFactory.getLog(MatrixMultiplyTest.class.getName());
+
+	private final static String TEST_DIR = "component/matrix/nativeBLAS/";
+
+	protected String getTestClassDir() {
+		return getTestDir() + this.getClass().getSimpleName() + "/";
+	}
+
+	protected String getTestName() {
+		return "matrixMultiplyOpenBLAS";
+	}
+
+	protected String getTestDir() {
+		return TEST_DIR;
+	}
+
+    public MatrixMultiplyOpenBLASTest(int i, int j, int k, double s, double s2, int p, boolean self) {
+		super(i,j,k,s,s2,p,self);
+	}
+
+	// TODO: test not working with native BLAS
+	@Override
+	public void testLeftNonContiguous(){}
+
+	@Override
+	protected File getConfigTemplateFile() {
+		return new File("./src/test/config/SystemDS-config-OpenBLAS.xml");
+	}
+
+	@Override
+	public void setUp() {
+		try {
+			String testname = getTestName() + String.valueOf(Math.random()*5);
+			addTestConfiguration(testname, new TestConfiguration(getTestClassDir(), testname));
+			loadTestConfiguration(getTestConfiguration(testname));
+
+			DMLConfig conf = new DMLConfig(getCurConfigFile().getPath());
+			ConfigurationManager.setLocalConfig(conf);
+
+			assertEquals(true, NativeHelper.isNativeLibraryLoaded());
+		} catch(Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
+
+	@Override
+	public void tearDown() {
+		TestUtils.clearDirectory(getCurLocalTempDir().getPath());
+		TestUtils.removeDirectories(new String[]{getCurLocalTempDir().getPath()});
+	}
+}