[SYSTEMDS-3928] Second-order Builtin Function for Independent Subnet Training

src/main/java/org/apache/sysds/common/Builtins.java

@@ -207,6 +207,7 @@ public enum Builtins {
 	ISNA("is.na", "isNA", false),
 	ISNAN("is.nan", "isNaN", false),
 	ISINF("is.infinite", "isInf", false),
+    ISN_TRAIN("independentSubnetTrain", true),
 	KM("km", true),
 	KMEANS("kmeans", true),
 	KMEANSPREDICT("kmeansPredict", true),

src/main/java/org/apache/sysds/runtime/controlprogram/context/ExecutionContext.java

@@ -810,7 +810,7 @@ public void unpinVariables(List<String> varList, Queue<Boolean> varsState) {
 		for (String varName : varList) {
 			Data dat = _variables.get(varName);
 			if (dat instanceof CacheableData<?>)
-				((CacheableData<?>)dat).enableCleanup(varsState.poll());
+				((CacheableData<?>)dat).enableCleanup(Boolean.TRUE.equals(varsState.poll()));
 			else if (dat instanceof ListObject)
 				((ListObject)dat).enableCleanup(varsState);
 		}

src/main/java/org/apache/sysds/runtime/instructions/cp/ListObject.java

@@ -552,7 +552,7 @@ public void enableCleanup(boolean flag) {
 	public void enableCleanup(Queue<Boolean> flags) {
 		for (Data dat : this.getData()) {
 			if (dat instanceof CacheableData<?>)
-				((CacheableData<?>)dat).enableCleanup(flags.poll());
+				((CacheableData<?>)dat).enableCleanup(Boolean.TRUE.equals(flags.poll()));
 			else if (dat instanceof ListObject)
 				((ListObject)dat).enableCleanup(flags);
 		}

src/test/config/SystemDS-config.xml

@@ -18,7 +18,7 @@
 -->
 
 <root>
-   <!-- The number of theads for the spark instance artificially selected-->
+   <!-- The number of threads for the spark instance artificially selected-->
    <sysds.local.spark.number.threads>2</sysds.local.spark.number.threads>
    <!-- The timeout of the federated tests to initialize the federated matrixes -->
    <sysds.federated.initialization.timeout>2</sysds.federated.initialization.timeout>

src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinIndSubnetTest.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.functions.builtin.part1;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+
+@RunWith(value = Parameterized.class)
+@net.jcip.annotations.NotThreadSafe
+public class BuiltinIndSubnetTest extends AutomatedTestBase {
+
+	private static final Log LOG = LogFactory.getLog(BuiltinIndSubnetTest.class.getName());
+
+	protected final static String TEST_NAME = "indSubnetTest_mnist_lenet";
+	protected final static String TEST_DIR = "functions/builtin/";
+	protected String TEST_CLASS_DIR = TEST_DIR + BuiltinIndSubnetTest.class.getSimpleName() + "/";
+
+	private final String dataset_path;
+	private final double least_expected_acc;
+	private final String out_path;
+
+	public BuiltinIndSubnetTest(String dataset_path, double least_expected_acc, String out_path) {
+		this.dataset_path = dataset_path;
+		this.least_expected_acc = least_expected_acc;
+		this.out_path = out_path;
+	}
+
+	@Parameters
+	public static Collection<Object[]> data() {
+		String path = "src/test/resources/datasets/MNIST/mnist_test.csv";
+		double least_expected_acc = 0.5;
+		String out_path = "accuracy";
+		List<Object[]> tests = new ArrayList<>();
+		tests.add(new Object[]{path, least_expected_acc, out_path});
+
+		return tests;
+	}
+
+	@Override
+	public void setUp() {
+		addTestConfiguration(TEST_CLASS_DIR, TEST_NAME);
+	}
+
+	@Test
+	public void testClassificationFit() {
+
+		getAndLoadTestConfiguration(TEST_NAME);
+
+		List<String> proArgs = new ArrayList<>();
+		proArgs.add("-args");
+		proArgs.add(this.dataset_path);
+		proArgs.add(output(this.out_path));
+
+		programArgs = proArgs.toArray(new String[proArgs.size()]);
+
+		fullDMLScriptName = getScript();
+
+		LOG.error(runTest(null));
+
+		double[][] from_DML = TestUtils.convertHashMapToDoubleArray(readDMLScalarFromOutputDir(this.out_path));
+		double accuracy = from_DML[0][0];
+		assertTrue("Accuracy lower than expected", accuracy > this.least_expected_acc);
+	}
+}