Bblfsh client upgrade

build.gradle

@@ -19,7 +19,7 @@ dependencies {
     implementation group: 'org.slf4j', name: 'slf4j-log4j12', version: '1.7.26'
     implementation group: 'org.apache.commons', name: 'commons-collections4', version: '4.2'
     implementation 'org.codehaus.groovy:groovy-all:2.4.15'
-    implementation 'org.bblfsh:bblfsh-client:1.11.0'
+    implementation 'org.bblfsh:bblfsh-client:2.1.0'
     implementation group: 'org.kohsuke', name: 'github-api', version: '1.101'
     implementation group: 'org.eclipse.jgit', name: 'org.eclipse.jgit', version: '5.1.3.201810200350-r'
     implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.9'
@@ -50,4 +50,4 @@ dockerCompose {
     useComposeFiles = ['./docker-compose.yml']
     dockerComposeWorkingDirectory = './docker/'
     captureContainersOutput = true
-}
+}

src/main/groovy/com/codebrig/arthur/SourceNode.groovy

@@ -2,9 +2,9 @@ package com.codebrig.arthur
 
 import com.codebrig.arthur.observe.structure.StructureLiteral
 import com.codebrig.arthur.observe.structure.StructureNaming
-import gopkg.in.bblfsh.sdk.v1.uast.generated.Node
-import gopkg.in.bblfsh.sdk.v1.uast.generated.Role
+import gopkg.in.bblfsh.sdk.v1.uast.role.generated.Role
 import org.apache.commons.collections4.iterators.TransformIterator
+import org.bblfsh.client.v2.*
 import scala.collection.JavaConverters
 
 /**
@@ -17,21 +17,21 @@ import scala.collection.JavaConverters
 class SourceNode {
 
     private final SourceLanguage language
-    private final Node rootNode
-    private final Node underlyingNode
+    private final JNode rootNode
+    private final JNode underlyingNode
     private final SourceNode parentSourceNode
     private final StructureNaming naming
     private final StructureLiteral literal
 
-    SourceNode(SourceLanguage language, Node underlyingNode) {
+    SourceNode(SourceLanguage language, JNode underlyingNode) {
         this(language, underlyingNode, underlyingNode)
     }
 
-    SourceNode(SourceLanguage language, Node rootNode, Node underlyingNode) {
+    SourceNode(SourceLanguage language, JNode rootNode, JNode underlyingNode) {
         this(language, rootNode, underlyingNode, null)
     }
 
-    SourceNode(SourceLanguage language, Node rootNode, Node underlyingNode, SourceNode parentNode) {
+    SourceNode(SourceLanguage language, JNode rootNode, JNode underlyingNode, SourceNode parentNode) {
         this.language = Objects.requireNonNull(language)
         this.rootNode = Objects.requireNonNull(rootNode)
         this.underlyingNode = Objects.requireNonNull(underlyingNode)
@@ -44,15 +44,19 @@ class SourceNode {
         return language
     }
 
-    Node getRootNode() {
+    JNode getRootNode() {
         return rootNode
     }
 
     SourceNode getRootSourceNode() {
         return new SourceNode(language, rootNode)
     }
 
-    Node getUnderlyingNode() {
+    /**
+     * todo: remove
+     */
+    @Deprecated
+    JNode getUnderlyingNode() {
         return underlyingNode
     }
 
@@ -61,14 +65,38 @@ class SourceNode {
     }
 
     String getInternalType() {
-        return underlyingNode.internalType()
+        for (int i = 0; i < underlyingNode.size(); i++) {
+            if (underlyingNode.keyAt(i) == "@type") {
+                return (underlyingNode.valueAt(i) as JString).str().replace(language.key + ":", "")
+            }
+        }
+        return "unknown"
     }
 
     boolean hasName() {
+        for (int i = 0; i < underlyingNode.size(); i++) {
+            if (underlyingNode.keyAt(i) == "Name") {
+                return true
+            }
+        }
         return naming.isNamedNodeType(this)
     }
 
     String getName() {
+        for (int i = 0; i < underlyingNode.size(); i++) {
+            if (underlyingNode.keyAt(i) == "Name") {
+                if (underlyingNode.valueAt(i) instanceof JObject) {
+                    def jobject = (underlyingNode.valueAt(i) as JObject)
+                    for (int z = 0; z < jobject.size(); z++) {
+                        if (jobject.keyAt(z) == "Name") {
+                            return (jobject.valueAt(z) as JString).str()
+                        }
+                    }
+                } else {
+                    return (underlyingNode.valueAt(i) as JString).str()
+                }
+            }
+        }
         return naming.getNodeName(this)
     }
 
@@ -82,8 +110,8 @@ class SourceNode {
     }
 
     Iterator<SourceNode> getChildren() {
-        def itr = asJavaIterator(underlyingNode.children())
-        return new TransformIterator<Node, SourceNode>(itr, { Node node ->
+        def itr = asJavaIterator(BblfshClient$.MODULE$.iterator(underlyingNode, new BblfshClient.ChildrenOrder$()).seq())
+        return new TransformIterator<JNode, SourceNode>(itr, { JNode node ->
             if (node == null) {
                 return null
             }
@@ -95,8 +123,48 @@ class SourceNode {
         return asJavaMap(underlyingNode.properties())
     }
 
-    Iterator<Role> getRoles() {
-        return asJavaIterator(underlyingNode.roles())
+    List<Role> getRoles() {
+        def roleIndex = -1
+        for (int i = 0; i < underlyingNode.size(); i++) {
+            if (underlyingNode.keyAt(i) == "@role") {
+                roleIndex = i
+                break
+            }
+        }
+
+        def roleList = new ArrayList<Role>()
+        if (roleIndex == -1) {
+            return roleList
+            //throw new IllegalStateException("Could not find @role")
+        }
+
+        def roles = underlyingNode.valueAt(roleIndex) as JArray
+        for (int i = 0; i < roles.size(); i++) {
+            //todo: better
+            def roleName = (roles.valueAt(i) as JString).str().toUpperCase()
+            switch (roleName) {
+                case "DOWHILE":
+                    roleName = "DO_WHILE"
+                    break
+                case "GREATERTHAN":
+                    roleName = "GREATER_THAN"
+                    break
+                case "GREATERTHANOREQUAL":
+                    roleName = "GREATER_THAN_OR_EQUAL"
+                    break
+                case "LEFTSHIFT":
+                    roleName = "LEFT_SHIFT"
+                    break
+                case "LESSTHAN":
+                    roleName = "LESS_THAN"
+                    break
+                case "LESSTHANOREQUAL":
+                    roleName = "LESS_THAN_OR_EQUAL"
+                    break
+            }
+            roleList += Eval.me("return new gopkg.in.bblfsh.sdk.v1.uast.role.generated.Role." + roleName + "\$()")
+        }
+        return roleList
     }
 
     boolean isLiteralNode() {
@@ -124,6 +192,10 @@ class SourceNode {
         return JavaConverters.asJavaCollectionConverter(scalaIterator).asJavaCollection().iterator()
     }
 
+    static <T> Iterator<T> asJavaIterator(scala.collection.Iterator<T> scalaIterator) {
+        return JavaConverters.asJavaIteratorConverter(scalaIterator).asJava()
+    }
+
     static Map<String, String> asJavaMap(scala.collection.Map<String, String> scalaMap) {
         return JavaConverters.mapAsJavaMapConverter(scalaMap).asJava()
     }

src/main/groovy/com/codebrig/arthur/generator/SchemaGenerator.groovy

@@ -6,11 +6,10 @@ import com.codebrig.arthur.observe.ObservationConfig
 import com.codebrig.arthur.observe.ObservedLanguage
 import com.codebrig.arthur.observe.structure.StructureFilter
 import com.codebrig.arthur.observe.structure.filter.WildcardFilter
-import gopkg.in.bblfsh.sdk.v1.protocol.generated.Encoding
-import gopkg.in.bblfsh.sdk.v1.protocol.generated.ParseResponse
+import gopkg.in.bblfsh.sdk.v2.protocol.driver.ParseResponse
 import groovy.transform.Canonical
 import groovy.transform.TupleConstructor
-import org.bblfsh.client.BblfshClient
+import org.bblfsh.client.v2.BblfshClient
 import org.eclipse.jgit.api.Git
 import org.kohsuke.github.GHDirection
 import org.kohsuke.github.GHRepositorySearchBuilder
@@ -139,8 +138,7 @@ class SchemaGenerator {
             log.info "Parsing: " + file
             def fileResponse = new FileParseResponse(file)
             def task = executorService.submit({
-                fileResponse.parseResponse = client.parse(
-                        file.name, file.text, observedLanguage.language.babelfishName, Encoding.UTF8$.MODULE$)
+                fileResponse.parseResponse = client.parse(file.name, file.text, observedLanguage.language.babelfishName)
                 return fileResponse
             } as Callable<FileParseResponse>)
             try {
@@ -151,17 +149,13 @@ class SchemaGenerator {
             }
         }).map({
             if (it instanceof FileParseResponse) {
-                if (it.parseResponse.status().isOk()) {
-                    def rootSourceNode = new SourceNode(observedLanguage.language, it.parseResponse.uast)
-                    if (filter.evaluate(rootSourceNode)) {
-                        observeSourceNode(observedLanguage, rootSourceNode)
-                    }
-                    extractSchema(observedLanguage, rootSourceNode)
-                    parseCount.getAndIncrement()
-                } else {
-                    log.error "Failed to parse: " + it.parsedFile + " - Reason: " + it.parseResponse.errors().toList().toString()
-                    failCount.getAndIncrement()
+                def rootNode = new BblfshClient.UastMethods(it.parseResponse.uast()).decode().root().load()
+                def rootSourceNode = new SourceNode(observedLanguage.language, rootNode)
+                if (filter.evaluate(rootSourceNode)) {
+                    observeSourceNode(observedLanguage, rootSourceNode)
                 }
+                extractSchema(observedLanguage, rootSourceNode)
+                parseCount.getAndIncrement()
             }
         }).count()
         executorService.shutdown()

src/main/groovy/com/codebrig/arthur/observe/ObservedLanguage.groovy

@@ -8,7 +8,7 @@ import com.codebrig.arthur.observe.observations.ObservedRoles
 import com.codebrig.arthur.observe.structure.StructureLiteral
 import com.google.common.base.CaseFormat
 import com.google.common.collect.Maps
-import gopkg.in.bblfsh.sdk.v1.uast.generated.Role
+import gopkg.in.bblfsh.sdk.v1.uast.role.generated.Role
 
 import java.util.stream.Collectors
 
@@ -107,18 +107,18 @@ class ObservedLanguage {
         }
     }
 
-    void observeRoles(String entity, Iterator<Role> entityRoles) {
+    void observeRoles(String entity, List<Role> entityRoles) {
         entity = toValidEntity(entity)
         roles.putIfAbsent(entity, new ObservedRoles())
-        roles.get(entity).observe(entityRoles.toList().stream()
+        roles.get(entity).observe(entityRoles.stream()
                 .map({ it -> it.name() })
                 .collect(Collectors.toList()).iterator())
     }
 
-    void observeRoles(String entity, List<String> entityRoles) {
+    void observeRoles(String entity, Iterator<String> entityRoles) {
         entity = toValidEntity(entity)
         roles.putIfAbsent(entity, new ObservedRoles())
-        roles.get(entity).observe(entityRoles.iterator())
+        roles.get(entity).observe(entityRoles)
     }
 
     void addEntityExtends(String entity) {

src/main/groovy/com/codebrig/arthur/observe/structure/StructureFilter.groovy

@@ -2,9 +2,9 @@ package com.codebrig.arthur.observe.structure
 
 import com.codebrig.arthur.SourceLanguage
 import com.codebrig.arthur.SourceNode
-import gopkg.in.bblfsh.sdk.v1.uast.generated.Node
 import org.apache.commons.collections4.Predicate
 import org.apache.commons.collections4.iterators.FilterIterator
+import org.bblfsh.client.v2.JNode
 
 /**
  * Used to filter through SourceNodes
@@ -40,7 +40,7 @@ abstract class StructureFilter<T extends StructureFilter, P> implements Predicat
         return new FilterIterator(sourceNodes, this)
     }
 
-    Iterator<SourceNode> getFilteredNodes(SourceLanguage language, Node node) {
+    Iterator<SourceNode> getFilteredNodes(SourceLanguage language, JNode node) {
         return getFilteredNodes(new SourceNode(language, node))
     }
 

src/main/groovy/com/codebrig/arthur/observe/structure/StructureNaming.groovy

@@ -1,6 +1,7 @@
 package com.codebrig.arthur.observe.structure
 
 import com.codebrig.arthur.SourceNode
+import com.codebrig.arthur.observe.structure.filter.TypeFilter
 
 /**
  * Used to get the names/qualified names of UAST nodes
@@ -9,13 +10,25 @@ import com.codebrig.arthur.SourceNode
  * @since 0.2
  * @author <a href="mailto:brandon.fergerson@codebrig.com">Brandon Fergerson</a>
  */
-trait StructureNaming {
+abstract class StructureNaming {
 
     boolean isNamedNodeType(SourceNode node) {
-        return isNamedNodeType(node.internalType)
+        def internalType = node.internalType
+        switch (internalType) {
+            case "uast:FunctionGroup":
+                return true
+        }
+        return isNamedNodeType(internalType)
     }
 
     abstract boolean isNamedNodeType(String internalType)
 
-    abstract String getNodeName(SourceNode node)
+    String getNodeName(SourceNode node) {
+        switch (node.internalType) {
+            case "uast:FunctionGroup":
+                return new TypeFilter("uast:Identifier").getFilteredNodes(node).next().getName()
+            default:
+                throw new IllegalArgumentException("Unsupported node type: " + node.internalType)
+        }
+    }
 }

src/main/groovy/com/codebrig/arthur/observe/structure/filter/FunctionFilter.groovy

@@ -15,9 +15,12 @@ class FunctionFilter extends StructureFilter<FunctionFilter, Void> {
     private final MultiFilter filter
 
     FunctionFilter() {
-        filter = MultiFilter.matchAll(
-                new RoleFilter("DECLARATION"), new RoleFilter("FUNCTION"),
-                new RoleFilter().reject("ARGUMENT", "RETURN", "INCOMPLETE", "BODY")
+        filter = MultiFilter.matchAny(
+                new TypeFilter("uast:FunctionGroup"),
+                MultiFilter.matchAll(
+                        new RoleFilter("DECLARATION"), new RoleFilter("FUNCTION"),
+                        new RoleFilter().reject("ARGUMENT", "RETURN", "INCOMPLETE", "BODY")
+                )
         )
     }
 

src/main/groovy/com/codebrig/arthur/observe/structure/filter/loop/ForLoopFilter.groovy

@@ -2,7 +2,6 @@ package com.codebrig.arthur.observe.structure.filter.loop
 
 import com.codebrig.arthur.SourceNode
 import com.codebrig.arthur.observe.structure.StructureFilter
-import com.codebrig.arthur.observe.structure.filter.InternalRoleFilter
 import com.codebrig.arthur.observe.structure.filter.MultiFilter
 import com.codebrig.arthur.observe.structure.filter.RoleFilter
 
@@ -26,15 +25,6 @@ class ForLoopFilter extends StructureFilter<ForLoopFilter, Void> {
 
     @Override
     boolean evaluate(SourceNode node) {
-        boolean result = filter.evaluate(node)
-        if (result) {
-            def matched = MultiFilter.matchAll(
-                    new InternalRoleFilter("Init", "initializers")
-            ).getFilteredNodes(node.children)
-            if (matched.hasNext()) {
-                return true
-            }
-        }
-        return result
+        return filter.evaluate(node)
     }
 }

src/main/groovy/com/codebrig/arthur/observe/structure/naming/BashNaming.groovy

@@ -13,7 +13,7 @@ import com.codebrig.arthur.observe.structure.filter.TypeFilter
  * @since 0.4
  * @author <a href="mailto:valpecaoco@gmail.com"> Val Pecaoco</a>
  */
-class BashNaming implements StructureNaming {
+class BashNaming extends StructureNaming {
 
     @Override
     boolean isNamedNodeType(String internalType) {
@@ -34,7 +34,7 @@ class BashNaming implements StructureNaming {
             case "var-def-element":
                 return getVarDefElementName(node)
             default:
-                throw new IllegalArgumentException("Unsupported Bash node type: " + node.internalType)
+                return super.getNodeName(node)
         }
     }