diff --git a/.gitignore b/.gitignore index d2fcdb9a4de..56ea870300e 100644 --- a/.gitignore +++ b/.gitignore @@ -143,6 +143,7 @@ scripts/perftest/fed/temp src/test/scripts/functions/iogen/*.raw src/test/scripts/functions/pipelines/intermediates/regression/* src/test/scripts/functions/pipelines/intermediates/classification/* +src/test/scripts/functions/dnn/nativeBLAS/* venv venv/* @@ -156,4 +157,4 @@ docker/mountFolder/*.bin docker/mountFolder/*.bin.mtd SEAL-*/ - +OpenBLAS-*/ diff --git a/docker/testsysds.Dockerfile b/docker/testsysds.Dockerfile index a4bfc144164..27a57bb9c88 100644 --- a/docker/testsysds.Dockerfile +++ b/docker/testsysds.Dockerfile @@ -18,8 +18,8 @@ # under the License. # #------------------------------------------------------------- -# Stage 1: Build SEAL -FROM ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 AS seal-build +# Stage 1: Build SEAL, OpenBLAS, MKL +FROM ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 AS build RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -28,18 +28,52 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ tar \ git \ ca-certificates \ + gnupg \ + python3 \ + python3-pip \ + python3-venv \ && rm -rf /var/lib/apt/lists/* WORKDIR /seal # Install SEAL -RUN wget -qO- https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz | tar xzf - \ - && cd SEAL-3.7.0 \ +ARG SEAL_VERSION="3.7.0" +RUN wget -qO- https://github.com/microsoft/SEAL/archive/refs/tags/v${SEAL_VERSION}.tar.gz | tar xzf - \ + && cd SEAL-${SEAL_VERSION} \ && cmake -S . -B build -DBUILD_SHARED_LIBS=ON \ && cmake --build build \ && cmake --install build --prefix /seal-install -# Stage 2: Final image with R, JDK, Maven, SEAL +WORKDIR /openBLAS + +# Install OpenBLAS +ARG OPENBLAS_VERSION="0.3.26" +RUN wget -qO- https://github.com/OpenMathLib/OpenBLAS/archive/refs/tags/v${OPENBLAS_VERSION}.tar.gz | tar xzf - \ + && cd OpenBLAS-${OPENBLAS_VERSION} \ + && make -j$(nproc) \ + && make install PREFIX=/openBLAS-install + +WORKDIR /mkl + +# Install old MKL, since SystemDS depends on intel MKL <= 2019.x +# Only package distro which has such old version available is PyPi +ENV PATH="/mkl-install/bin:$PATH" +RUN python3 -m venv /mkl-install && \ + pip install mkl-devel==2018.0.0 + +# Delete unused libraries, since MKL libraries take up a lot of disk space +RUN find /mkl-install/lib \( \ + -name '*ilp64*' -o \ + -name 'libmkl_gnu_thread*' -o \ + -name 'libmkl_tbb_thread*' -o \ + -name '*_openmpi_*' -o \ + -name '*_intelmpi_*' -o \ + -name 'libmkl_vml*' -o \ + -name 'libmkl_scalapack*' \ + \) -delete + + +# Stage 2: Final image with R, JDK, Maven, SEAL, OpenBLAS, MKL FROM ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 WORKDIR /usr/src/ @@ -71,6 +105,7 @@ RUN apt-get install -y --no-install-recommends \ libssl-dev \ r-base-dev \ r-base-core \ + gfortran \ && apt-get clean && rm -rf /var/lib/apt/lists/* \ && mkdir -p /usr/lib/jvm \ && wget -qO- \ @@ -101,8 +136,16 @@ RUN mkdir -p $HADOOP_HOME/lib/native \ rm -rf native # Copy SEAL -COPY --from=seal-build /seal-install/lib/ /usr/local/lib/ -COPY --from=seal-build /seal-install/include/ /usr/local/include/ +COPY --from=build /seal-install/lib/ /usr/local/lib/ +COPY --from=build /seal-install/include/ /usr/local/include/ + +# Copy OpenBLAS +COPY --from=build /openBLAS-install/lib/ /usr/local/lib/ +COPY --from=build /openBLAS-install/include/ /usr/local/include/ + +# Copy MKL +COPY --from=build /mkl-install/include /usr/local/include/ +COPY --from=build /mkl-install/lib /usr/local/lib/ ENV LD_LIBRARY_PATH=/opt/hadoop/lib/native;/usr/local/lib/ diff --git a/src/test/config/SystemDS-config-MKL.xml b/src/test/config/SystemDS-config-MKL.xml new file mode 100644 index 00000000000..a609f9d7d4c --- /dev/null +++ b/src/test/config/SystemDS-config-MKL.xml @@ -0,0 +1,23 @@ + + + + mkl + /usr/local/lib + \ No newline at end of file diff --git a/src/test/config/SystemDS-config-OpenBLAS.xml b/src/test/config/SystemDS-config-OpenBLAS.xml new file mode 100644 index 00000000000..0de1d705e56 --- /dev/null +++ b/src/test/config/SystemDS-config-OpenBLAS.xml @@ -0,0 +1,23 @@ + + + + openblas + /usr/local/lib + \ No newline at end of file diff --git a/src/test/java/org/apache/sysds/test/component/matrix/MatrixMultiplyTest.java b/src/test/java/org/apache/sysds/test/component/matrix/MatrixMultiplyTest.java index 0934898bcc2..c88e98aecd9 100644 --- a/src/test/java/org/apache/sysds/test/component/matrix/MatrixMultiplyTest.java +++ b/src/test/java/org/apache/sysds/test/component/matrix/MatrixMultiplyTest.java @@ -28,19 +28,17 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.sysds.runtime.functionobjects.Multiply; -import org.apache.sysds.runtime.functionobjects.Plus; import org.apache.sysds.runtime.matrix.data.MatrixBlock; -import org.apache.sysds.runtime.matrix.operators.AggregateBinaryOperator; -import org.apache.sysds.runtime.matrix.operators.AggregateOperator; +import org.apache.sysds.runtime.matrix.data.LibMatrixNative; import org.apache.sysds.test.TestUtils; +import org.apache.sysds.test.AutomatedTestBase; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; @RunWith(value = Parameterized.class) -public class MatrixMultiplyTest { +public class MatrixMultiplyTest extends AutomatedTestBase{ protected static final Log LOG = LogFactory.getLog(MatrixMultiplyTest.class.getName()); // left side @@ -278,9 +276,16 @@ private static String size(MatrixBlock a) { } private static MatrixBlock multiply(MatrixBlock a, MatrixBlock b, int k) { - AggregateOperator agg = new AggregateOperator(0, Plus.getPlusFnObject()); - AggregateBinaryOperator mult = new AggregateBinaryOperator(Multiply.getMultiplyFnObject(), agg, k); - return a.aggregateBinaryOperations(a, b, mult); + MatrixBlock ret = new MatrixBlock(); + try { + LibMatrixNative.matrixMult(a,b,ret,k); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + return ret; } + @Override + public void setUp(){} } diff --git a/src/test/java/org/apache/sysds/test/component/matrix/nativeBLAS/MatrixMultiplyMKLTest.java b/src/test/java/org/apache/sysds/test/component/matrix/nativeBLAS/MatrixMultiplyMKLTest.java new file mode 100644 index 00000000000..e16cc0f07cd --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/matrix/nativeBLAS/MatrixMultiplyMKLTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysds.test.component.matrix.nativeBLAS; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.File; +import java.lang.Math; +import java.lang.String; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.utils.NativeHelper; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.apache.sysds.test.component.matrix.MatrixMultiplyTest; + +public class MatrixMultiplyMKLTest extends MatrixMultiplyTest{ + protected static final Log LOG = LogFactory.getLog(MatrixMultiplyTest.class.getName()); + + private final static String TEST_DIR = "component/matrix/nativeBLAS/"; + + protected String getTestClassDir() { + return getTestDir() + this.getClass().getSimpleName() + "/"; + } + + protected String getTestName() { + return "matrixMultiplyMKL"; + } + + protected String getTestDir() { + return TEST_DIR; + } + + public MatrixMultiplyMKLTest(int i, int j, int k, double s, double s2, int p, boolean self) { + super(i,j,k,s,s2,p,self); + } + + // TODO: test not working with native BLAS + @Override + public void testLeftNonContiguous(){} + + @Override + protected File getConfigTemplateFile() { + return new File("./src/test/config/SystemDS-config-MKL.xml"); + } + + @Override + public void setUp() { + try { + String testname = getTestName() + String.valueOf(Math.random()*5); + addTestConfiguration(testname, new TestConfiguration(getTestClassDir(), testname)); + loadTestConfiguration(getTestConfiguration(testname)); + + DMLConfig conf = new DMLConfig(getCurConfigFile().getPath()); + ConfigurationManager.setLocalConfig(conf); + + assertEquals(true, NativeHelper.isNativeLibraryLoaded()); + } catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Override + public void tearDown() { + TestUtils.clearDirectory(getCurLocalTempDir().getPath()); + TestUtils.removeDirectories(new String[]{getCurLocalTempDir().getPath()}); + } +} diff --git a/src/test/java/org/apache/sysds/test/component/matrix/nativeBLAS/MatrixMultiplyOpenBLASTest.java b/src/test/java/org/apache/sysds/test/component/matrix/nativeBLAS/MatrixMultiplyOpenBLASTest.java new file mode 100644 index 00000000000..deea31cae7c --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/matrix/nativeBLAS/MatrixMultiplyOpenBLASTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysds.test.component.matrix.nativeBLAS; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.File; +import java.lang.Math; +import java.lang.String; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.utils.NativeHelper; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.apache.sysds.test.component.matrix.MatrixMultiplyTest; + +public class MatrixMultiplyOpenBLASTest extends MatrixMultiplyTest{ + protected static final Log LOG = LogFactory.getLog(MatrixMultiplyTest.class.getName()); + + private final static String TEST_DIR = "component/matrix/nativeBLAS/"; + + protected String getTestClassDir() { + return getTestDir() + this.getClass().getSimpleName() + "/"; + } + + protected String getTestName() { + return "matrixMultiplyOpenBLAS"; + } + + protected String getTestDir() { + return TEST_DIR; + } + + public MatrixMultiplyOpenBLASTest(int i, int j, int k, double s, double s2, int p, boolean self) { + super(i,j,k,s,s2,p,self); + } + + // TODO: test not working with native BLAS + @Override + public void testLeftNonContiguous(){} + + @Override + protected File getConfigTemplateFile() { + return new File("./src/test/config/SystemDS-config-OpenBLAS.xml"); + } + + @Override + public void setUp() { + try { + String testname = getTestName() + String.valueOf(Math.random()*5); + addTestConfiguration(testname, new TestConfiguration(getTestClassDir(), testname)); + loadTestConfiguration(getTestConfiguration(testname)); + + DMLConfig conf = new DMLConfig(getCurConfigFile().getPath()); + ConfigurationManager.setLocalConfig(conf); + + assertEquals(true, NativeHelper.isNativeLibraryLoaded()); + } catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Override + public void tearDown() { + TestUtils.clearDirectory(getCurLocalTempDir().getPath()); + TestUtils.removeDirectories(new String[]{getCurLocalTempDir().getPath()}); + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/dnn/nativeBLAS/Conv2DMKLTest.java b/src/test/java/org/apache/sysds/test/functions/dnn/nativeBLAS/Conv2DMKLTest.java new file mode 100644 index 00000000000..5d4a17654f7 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/dnn/nativeBLAS/Conv2DMKLTest.java @@ -0,0 +1,46 @@ +package org.apache.sysds.test.functions.dnn.nativeBLAS; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.File; + +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.apache.sysds.test.functions.dnn.Conv2DTest; +import org.apache.sysds.utils.NativeHelper; + +public class Conv2DMKLTest extends Conv2DTest { + private final static String TEST_NAME = "Conv2DTest"; + private final static String TEST_DIR = "functions/dnn/nativeBLAS/"; + private final static String TEST_CLASS_DIR = TEST_DIR + Conv2DTest.class.getSimpleName() + "/"; + + @Override + protected File getConfigTemplateFile() { + return new File("./src/test/config/SystemDS-config-MKL.xml"); + } + + @Override + public void setUp() { + try { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"B"})); + + loadTestConfiguration(getTestConfiguration(TEST_NAME)); + DMLConfig conf = new DMLConfig(getCurConfigFile().getPath()); + ConfigurationManager.setLocalConfig(conf); + assertEquals(true, NativeHelper.isNativeLibraryLoaded()); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Override + public void tearDown() { + TestUtils.clearDirectory(getCurLocalTempDir().getPath()); + TestUtils.removeDirectories(new String[]{getCurLocalTempDir().getPath()}); + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/dnn/nativeBLAS/Conv2DOpenBLASTest.java b/src/test/java/org/apache/sysds/test/functions/dnn/nativeBLAS/Conv2DOpenBLASTest.java new file mode 100644 index 00000000000..145e146301d --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/dnn/nativeBLAS/Conv2DOpenBLASTest.java @@ -0,0 +1,46 @@ +package org.apache.sysds.test.functions.dnn.nativeBLAS; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.File; + +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.apache.sysds.test.functions.dnn.Conv2DTest; +import org.apache.sysds.utils.NativeHelper; + +public class Conv2DOpenBLASTest extends Conv2DTest { + private final static String TEST_NAME = "Conv2DTest"; + private final static String TEST_DIR = "functions/dnn/nativeBLAS/"; + private final static String TEST_CLASS_DIR = TEST_DIR + Conv2DTest.class.getSimpleName() + "/"; + + @Override + protected File getConfigTemplateFile() { + return new File("./src/test/config/SystemDS-config-OpenBLAS.xml"); + } + + @Override + public void setUp() { + try { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"B"})); + + loadTestConfiguration(getTestConfiguration(TEST_NAME)); + DMLConfig conf = new DMLConfig(getCurConfigFile().getPath()); + ConfigurationManager.setLocalConfig(conf); + assertEquals(true, NativeHelper.isNativeLibraryLoaded()); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Override + public void tearDown() { + TestUtils.clearDirectory(getCurLocalTempDir().getPath()); + TestUtils.removeDirectories(new String[]{getCurLocalTempDir().getPath()}); + } +}