Merge pull request #141 from codellm-devkit/fix/239-240-duplicate-key-node-range-errors

Fix/239 240 duplicate key node range errors

Author: sinha108

src/main/java/com/ibm/cldk/SymbolTable.java

@@ -593,7 +593,7 @@ private static String getTypeErasureSignature(CallableDeclaration callableDecl)
                 Parameter parameter = (Parameter) param;
                 ResolvedType resolvedType = parameter.getType().resolve();
                 if (parameter.isVarArgs()) {
-                    erasureParameterTypes.add(resolvedType.describe() + "[]");
+                    erasureParameterTypes.add(resolvedType.erasure().describe() + "[]");
                 } else {
                     erasureParameterTypes.add(resolvedType.erasure().describe());
                 }
@@ -1147,21 +1147,44 @@ private static boolean excludeSourceRoot(Path sourceRoot) {
         return false;
     }
 
+    /**
+     * Sets up lexical preserving printer for the given compilation unit in a safe manner by checking
+     * whether any node in the unit is missing ranges, which can result in exception.
+     *
+     * @param compilationUnit Compilation unit to be set with lexical preserving printer
+     * @return compilation unit set up with lexical preserving printer or the original compilation
+     *         unit if the unit contains range-missing nodes
+     */
+    private static CompilationUnit safeLexicalPreservingPrinterSetup(CompilationUnit compilationUnit) {
+        // setup lexical-preserving printer only if CU has no missing-range nodes
+        boolean hasNodeWithMissingRange = compilationUnit.findAll(Node.class).stream()
+                .anyMatch(n -> !n.getRange().isPresent());
+        if (!hasNodeWithMissingRange) {
+            return LexicalPreservingPrinter.setup(compilationUnit);
+        }
+        return compilationUnit;
+    }
+
     public static Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>> extractAll(Path projectRootPath)
             throws IOException {
-        SymbolSolverCollectionStrategy symbolSolverCollectionStrategy = new SymbolSolverCollectionStrategy();
+        ParserConfiguration config = new ParserConfiguration()
+                .setStoreTokens(true)
+                .setAttributeComments(true)
+                .setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21);
+        SymbolSolverCollectionStrategy symbolSolverCollectionStrategy = new SymbolSolverCollectionStrategy(config);
         ProjectRoot projectRoot = symbolSolverCollectionStrategy.collect(projectRootPath);
         javaSymbolSolver = (JavaSymbolSolver) symbolSolverCollectionStrategy.getParserConfiguration()
-                .setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21).getSymbolResolver().get();
+                .getSymbolResolver().get();
         Map<String, JavaCompilationUnit> symbolTable = new LinkedHashMap<>();
         Map<String, List<Problem>> parseProblems = new HashMap<>();
         for (SourceRoot sourceRoot : projectRoot.getSourceRoots()) {
             if (excludeSourceRoot(sourceRoot.getRoot())) {
                 continue;
             }
+            sourceRoot.setParserConfiguration(config);
             for (ParseResult<CompilationUnit> parseResult : sourceRoot.tryToParse()) {
                 if (parseResult.isSuccessful()) {
-                    CompilationUnit compilationUnit = LexicalPreservingPrinter.setup(parseResult.getResult().get());
+                    CompilationUnit compilationUnit = safeLexicalPreservingPrinterSetup(parseResult.getResult().get());
                     symbolTable.put(compilationUnit.getStorage().get().getPath().toString(),
                             processCompilationUnit(compilationUnit));
                 } else {
@@ -1181,13 +1204,15 @@ public static Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>>
         combinedTypeSolver.add(new ReflectionTypeSolver());
 
         ParserConfiguration parserConfiguration = new ParserConfiguration()
+                .setStoreTokens(true)
+                .setAttributeComments(true)
                 .setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21);
         parserConfiguration.setSymbolResolver(new JavaSymbolSolver(combinedTypeSolver));
 
         JavaParser javaParser = new JavaParser(parserConfiguration);
         ParseResult<CompilationUnit> parseResult = javaParser.parse(code);
         if (parseResult.isSuccessful()) {
-            CompilationUnit compilationUnit = LexicalPreservingPrinter.setup(parseResult.getResult().get());
+            CompilationUnit compilationUnit = safeLexicalPreservingPrinterSetup(parseResult.getResult().get());
             Log.debug("Successfully parsed code. Now processing compilation unit");
             symbolTable.put("<pseudo-path>", processCompilationUnit(compilationUnit));
         } else {
@@ -1216,6 +1241,8 @@ public static Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>>
                 .getSymbolResolver().get();
         Log.info("Setting parser language level to JAVA_21");
         ParserConfiguration parserConfiguration = new ParserConfiguration()
+                .setStoreTokens(true)
+                .setAttributeComments(true)
                 .setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21);
         parserConfiguration.setSymbolResolver(javaSymbolSolver);
 
@@ -1229,7 +1256,7 @@ public static Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>>
         for (Path javaFilePath : javaFilePaths) {
             ParseResult<CompilationUnit> parseResult = javaParser.parse(javaFilePath);
             if (parseResult.isSuccessful()) {
-                CompilationUnit compilationUnit = LexicalPreservingPrinter.setup(parseResult.getResult().get());
+                CompilationUnit compilationUnit = safeLexicalPreservingPrinterSetup(parseResult.getResult().get());
                 System.out.println("Successfully parsed file: " + javaFilePath.toString());
                 symbolTable.put(compilationUnit.getStorage().get().getPath().toString(),
                         processCompilationUnit(compilationUnit));

src/test/java/com/ibm/cldk/SymbolTableTest.java

@@ -26,7 +26,7 @@ private String getJavaCodeForTestResource(String resourcePath) {
     }
 
     @Test
-    public void testExtractSingleGenricsDuplicateSignature() throws IOException {
+    public void testExtractSingleGenricsDuplicateSignature_Validate() throws IOException {
         String javaCode = getJavaCodeForTestResource("test-applications/generics-varargs-duplicate-signature-test/Validate.java");
         Map<String, JavaCompilationUnit> symbolTable = SymbolTable.extractSingle(javaCode).getLeft();
         Assertions.assertEquals(1, symbolTable.size());
@@ -36,6 +36,26 @@ public void testExtractSingleGenricsDuplicateSignature() throws IOException {
         Assertions.assertEquals(17, callables.size());
     }
 
+    @Test
+    public void testExtractSingleGenricsDuplicateSignature_FunctorUtils() throws IOException {
+        String javaCode = getJavaCodeForTestResource("test-applications/generics-varargs-duplicate-signature-test/FunctorUtils.java");
+        Map<String, JavaCompilationUnit> symbolTable = SymbolTable.extractSingle(javaCode).getLeft();
+        Assertions.assertEquals(1, symbolTable.size());
+        Map<String, Type> typeDeclaration = symbolTable.values().iterator().next().getTypeDeclarations();
+        Assertions.assertEquals(1, typeDeclaration.size());
+        Map<String, Callable> callables = typeDeclaration.values().iterator().next().getCallableDeclarations();
+        Assertions.assertEquals(10, callables.size());
+    }
+
+    @Test
+    public void testExtractSingleMissingNodeRange() throws IOException {
+        String javaCode = getJavaCodeForTestResource("test-applications/missing-node-range-test/WeakHashtableTestCase.java");
+        Map<String, JavaCompilationUnit> symbolTable = SymbolTable.extractSingle(javaCode).getLeft();
+        Assertions.assertEquals(1, symbolTable.size());
+        Map<String, Type> typeDeclaration = symbolTable.values().iterator().next().getTypeDeclarations();
+        Assertions.assertEquals(2, typeDeclaration.size());
+    }
+
     @Test
     public void testCallSiteArgumentExpression() throws IOException {
         String javaCode = getJavaCodeForTestResource("test-applications/generics-varargs-duplicate-signature-test/Validate.java");

src/test/resources/test-applications/generics-varargs-duplicate-signature-test/FunctorUtils.java

@@ -0,0 +1,76 @@
+import java.util.Collection;
+import java.util.Objects;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import org.apache.commons.collections4.Predicate;
+
+final class FunctorUtils {
+
+    private static <T> T[] clone(final T... array) {
+        return array != null ? array.clone() : null;
+    }
+
+    static <R extends java.util.function.Predicate<T>, P extends java.util.function.Predicate<? super T>, T> R coerce(final P predicate) {
+        return (R) predicate;
+    }
+
+    static <R extends Function<I, O>, P extends Function<? super I, ? extends O>, I, O> R coerce(final P transformer) {
+        return (R) transformer;
+    }
+
+    static <T extends Consumer<?>> T[] copy(final T... consumers) {
+        return clone(consumers);
+    }
+
+    static <T extends java.util.function.Predicate<?>> T[] copy(final T... predicates) {
+        return clone(predicates);
+    }
+
+    static <T extends Function<?, ?>> T[] copy(final T... transformers) {
+        return clone(transformers);
+    }
+
+    static <T> Predicate<? super T>[] validate(final Collection<? extends java.util.function.Predicate<? super T>> predicates) {
+        Objects.requireNonNull(predicates, "predicates");
+        // convert to array like this to guarantee iterator() ordering
+        @SuppressWarnings("unchecked") // OK
+        final Predicate<? super T>[] preds = new Predicate[predicates.size()];
+        int i = 0;
+        for (final java.util.function.Predicate<? super T> predicate : predicates) {
+            preds[i] = (Predicate<? super T>) predicate;
+            if (preds[i] == null) {
+                throw new NullPointerException("predicates[" + i + "]");
+            }
+            i++;
+        }
+        return preds;
+    }
+
+    static void validate(final Consumer<?>... consumers) {
+        Objects.requireNonNull(consumers, "consumers");
+        for (int i = 0; i < consumers.length; i++) {
+            if (consumers[i] == null) {
+                throw new NullPointerException("closures[" + i + "]");
+            }
+        }
+    }
+
+    static void validate(final Function<?, ?>... functions) {
+        Objects.requireNonNull(functions, "functions");
+        for (int i = 0; i < functions.length; i++) {
+            if (functions[i] == null) {
+                throw new NullPointerException("functions[" + i + "]");
+            }
+        }
+    }
+
+    static void validate(final java.util.function.Predicate<?>... predicates) {
+        Objects.requireNonNull(predicates, "predicates");
+        for (int i = 0; i < predicates.length; i++) {
+            if (predicates[i] == null) {
+                throw new NullPointerException("predicates[" + i + "]");
+            }
+        }
+    }
+
+}

src/test/resources/test-applications/missing-node-range-test/WeakHashtableTestCase.java

@@ -0,0 +1,36 @@
+import java.lang.ref.ReferenceQueue;
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import junit.framework.TestCase;
+
+public class WeakHashtableTestCase extends TestCase {
+
+    public static class TestThread extends Thread {
+
+        public TestThread(final String name) {
+            super(name);
+        }
+
+        @Override
+        public void run() {
+            for (int i = 0; i < RUN_LOOPS; i++) {
+                hashtable.put("key:" + i % 10, Boolean.TRUE);
+                if (i % 50 == 0) {
+                    yield();
+                }
+            }
+        }
+    }
+    private static final int RUN_LOOPS = 3000;
+    private static WeakHashtable hashtable;
+
+    public WeakHashtableTestCase(final String testName) {
+        super(testName);
+    }
+
+}