fix(interactive): Add Flex Type Benchmark

.github/workflows/interactive.yml

@@ -156,6 +156,9 @@ jobs:
         cp ${GITHUB_WORKSPACE}/flex/interactive/examples/new_graph_algo/import.yaml ${INTERACTIVE_WORKSPACE}/data/new_graph_algo/import.yaml
         mkdir -p ${INTERACTIVE_WORKSPACE}/data/modern_graph
         cp ${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml ${INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml
+        mkdir -p ${INTERACTIVE_WORKSPACE}/data/type_test
+        cp ${GITHUB_WORKSPACE}/interactive_engine/compiler/src/test/resources/flex_bench/modern.yaml ${INTERACTIVE_WORKSPACE}/data/type_test/graph.yaml
+        cp ${GITHUB_WORKSPACE}/interactive_engine/compiler/src/test/resources/flex_bench/import.yaml ${INTERACTIVE_WORKSPACE}/data/type_test/import.yaml
 
         # load graph
         cd ${GITHUB_WORKSPACE}/flex/build
@@ -169,6 +172,15 @@ jobs:
         GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/new_graph_algo/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/new_graph_algo/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/new_graph_algo/indices/
         export FLEX_DATA_DIR=../interactive/examples/modern_graph
         GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml -l ../interactive/examples/modern_graph/bulk_load.yaml -d ${INTERACTIVE_WORKSPACE}/data/modern_graph/indices/
+        export FLEX_DATA_DIR=${GITHUB_WORKSPACE}/interactive_engine/compiler/src/test/resources/flex_bench/data
+        GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/type_test/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/type_test/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/type_test/indices/
+
+    - name: Interactive Type Test
+      env:
+        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
+      run: |
+        cd ${GITHUB_WORKSPACE}/flex/tests/hqps
+        bash hqps_type_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml
 
     - name: Test HQPS admin http service
       env: 

flex/tests/hqps/hqps_type_test.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+# Copyright 2020 Alibaba Group Holding Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+FLEX_HOME=${SCRIPT_DIR}/../../
+BULK_LOADER=${FLEX_HOME}/build/bin/bulk_loader
+SERVER_BIN=${FLEX_HOME}/build/bin/interactive_server
+GIE_HOME=${FLEX_HOME}/../interactive_engine/
+
+# 
+if [ $# -ne 2 ]; then
+  echo "Receives: $# args, need 2 args"
+  echo "Usage: $0 <INTERACTIVE_WORKSPACE> <ENGINE_CONFIG>"
+  exit 1
+fi
+
+INTERACTIVE_WORKSPACE=$1
+ENGINE_CONFIG_PATH=$2
+
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+NC='\033[0m' # No Color
+err() {
+  echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] -ERROR- $* ${NC}" >&2
+}
+
+info() {
+  echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] -INFO- $* ${NC}"
+}
+
+
+kill_service(){
+    info "Kill Service first"
+    ps -ef | grep "com.alibaba.graphscope.GraphServer" | awk '{print $2}' | xargs kill -9 || true
+    ps -ef | grep "interactive_server" |  awk '{print $2}' | xargs kill -9  || true
+    sleep 3
+    # check if service is killed
+    info "Kill Service success"
+}
+
+# kill service when exit
+trap kill_service EXIT
+
+
+# start engine service and load ldbc graph
+start_engine_service(){
+    # suppose graph has been loaded, check ${GRAPH_CSR_DATA_DIR} exists
+
+    #check SERVER_BIN exists
+    if [ ! -f ${SERVER_BIN} ]; then
+        err "SERVER_BIN not found"
+        exit 1
+    fi
+    cmd="${SERVER_BIN} -c ${ENGINE_CONFIG_PATH} --start-compiler true "
+    cmd="${cmd} -w ${INTERACTIVE_WORKSPACE} --enable-admin-service true > /tmp/engine.log 2>&1 &"
+
+    info "Start engine service with command: ${cmd}"
+    ${cmd} &
+    sleep 5
+    #check interactive_server is running, if not, exit
+    ps -ef | grep "interactive_server" | grep -v grep
+
+    info "Start engine service success"
+}
+
+
+
+run_type_test() {
+    echo "run type test"
+    pushd ${GIE_HOME}/compiler
+    cmd="mvn test -Dtest=com.alibaba.graphscope.cypher.integration.flex.bench.FlexTypeTest"
+    info "Run type test with command: ${cmd}"
+    ${cmd}
+    info "Run type test success"
+    popd
+}
+
+kill_service
+start_engine_service
+run_type_test
+kill_service

flex/tests/hqps/interactive_config_test.yaml

@@ -12,13 +12,17 @@ compute_engine:
     type: file # file/sqlite/etcd
   wal_uri: file://{GRAPH_DATA_DIR}/wal # Could be file://{GRAPH_DATA_DIR}/wal or other supported storage class. GRAPH_DATA_DIR is the placeholder for the graph data directory.
 compiler:
+  physical:
+    opt:
+      config: proto
   planner:
     is_on: true
-    opt: RBO
+    opt: CBO
     rules:
       - FilterIntoJoinRule
       - FilterMatchRule
-      - NotMatchToAntiJoinRule
+      - ExtendIntersectRule
+      - ExpandGetVFusionRule
   meta:
     reader:
       schema:

interactive_engine/compiler/Makefile

@@ -25,6 +25,7 @@ physical:=
 procedure:=
 extra:=
 config.path:=conf/ir.compiler.properties
+disable.expr.simplify:=
 
 build:
 	cd $(CUR_DIR)/.. && \
@@ -63,6 +64,7 @@ run:
 	  -Dgraph.planner.rules=${graph.planner.rules} \
 	  -Dgraph.planner.opt=${graph.planner.opt} \
 	  -Dgraph.statistics=${graph.statistics} \
+	  -Ddisable.expr.simplify={disable.expr.simplify} \
 	  com.alibaba.graphscope.GraphServer ${config.path}
 
 # make physical_plan config.path='<path to the config file>'

interactive_engine/compiler/pom.xml

@@ -362,6 +362,7 @@
                         <exclude>**/IrPatternTest.java</exclude>
                         <exclude>**/MovieTest.java</exclude>
                         <exclude>**/GraphAlgoTest.java</exclude>
+                        <exclude>**/FlexTypeTest.java</exclude>
                     </excludes>
                 </configuration>
             </plugin>

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/FrontendConfig.java

@@ -66,4 +66,7 @@ public class FrontendConfig {
 
     public static final Config<Long> METRICS_TOOL_INTERVAL_MS =
             Config.longConfig("metrics.tool.interval.ms", 5 * 60 * 1000L);
+
+    public static final Config<Boolean> DISABLE_EXPR_SIMPLIFY =
+            Config.boolConfig("disable.expr.simplify", false);
 }

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java

@@ -52,7 +52,7 @@ public StaticIrMetaFetcher(IrMetaReader dataReader, List<IrMetaTracker> tracker)
         this.metaStats =
                 new IrMetaStats(
                         meta.getSnapshotId(), meta.getSchema(), meta.getStoredProcedures(), stats);
-        if (tracker != null && stats != null) {
+        if (tracker != null) {
             tracker.forEach(t -> t.onStatsChanged(this.metaStats));
         }
     }

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/glogue/calcite/handler/GraphCollationHandler.java

@@ -1,19 +1,17 @@
 /*
+ * Copyright 2020 Alibaba Group Holding Limited.
  *
- *  * Copyright 2020 Alibaba Group Holding Limited.
- *  *
- *  * Licensed under the Apache License, Version 2.0 (the "License");
- *  * you may not use this file except in compliance with the License.
- *  * You may obtain a copy of the License at
- *  *
- *  * http://www.apache.org/licenses/LICENSE-2.0
- *  *
- *  * Unless required by applicable law or agreed to in writing, software
- *  * distributed under the License is distributed on an "AS IS" BASIS,
- *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  * See the License for the specific language governing permissions and
- *  * limitations under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.alibaba.graphscope.common.ir.meta.glogue.calcite.handler;

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java

@@ -36,6 +36,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.math.BigDecimal;
 import java.util.Map;
 import java.util.Objects;
 
@@ -45,6 +46,15 @@
 public interface IrDataTypeConvertor<T> {
     Logger logger = LoggerFactory.getLogger(IrDataTypeConvertor.class);
 
+    // support unsigned type as decimal type with fixed precision and scale
+    int UINT32_PRECISION = 10;
+    int UINT32_SCALE = 0;
+    int UINT64_PRECISION = 20;
+    int UINT64_SCALE = 0;
+
+    BigDecimal UINT32_MAX = new BigDecimal("4294967295");
+    BigDecimal UINT64_MAX = new BigDecimal("18446744073709551615");
+
     RelDataType convert(T dataFrom);
 
     T convert(RelDataType dataFrom);
@@ -220,9 +230,15 @@ public RelDataType convert(GSDataTypeDesc from) {
                     case "DT_ANY":
                         // any type
                         return typeFactory.createSqlType(SqlTypeName.ANY);
+                    case "DT_UNSIGNED_INT32":
+                        return typeFactory.createSqlType(
+                                SqlTypeName.DECIMAL, UINT32_PRECISION, UINT32_SCALE);
                     case "DT_SIGNED_INT32":
                         // 4-bytes signed integer
                         return typeFactory.createSqlType(SqlTypeName.INTEGER);
+                    case "DT_UNSIGNED_INT64":
+                        return typeFactory.createSqlType(
+                                SqlTypeName.DECIMAL, UINT64_PRECISION, UINT64_SCALE);
                     case "DT_SIGNED_INT64":
                         // 8-bytes signed integer
                         return typeFactory.createSqlType(SqlTypeName.BIGINT);
@@ -435,12 +451,20 @@ public GSDataTypeDesc convert(RelDataType from) {
                                     ImmutableMap.of("key_type", keyType, "value_type", valueType));
                     break;
                 case DECIMAL:
-                    yamlDesc =
-                            ImmutableMap.of(
-                                    "decimal",
-                                    ImmutableMap.of(
-                                            "precision", from.getPrecision(),
-                                            "scale", from.getScale()));
+                    if (from.getPrecision() == UINT32_PRECISION
+                            && from.getScale() == UINT32_SCALE) {
+                        yamlDesc = ImmutableMap.of("primitive_type", "DT_UNSIGNED_INT32");
+                    } else if (from.getPrecision() == UINT64_PRECISION
+                            && from.getScale() == UINT64_SCALE) {
+                        yamlDesc = ImmutableMap.of("primitive_type", "DT_UNSIGNED_INT64");
+                    } else {
+                        yamlDesc =
+                                ImmutableMap.of(
+                                        "decimal",
+                                        ImmutableMap.of(
+                                                "precision", from.getPrecision(),
+                                                "scale", from.getScale()));
+                    }
                     break;
                 default:
                     if (throwsOnFail) {

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java

@@ -46,6 +46,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.math.BigDecimal;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -113,7 +114,26 @@ public static final Common.Value protoValue(RexLiteral literal) {
                                 : (String) literal.getValue();
                 return Common.Value.newBuilder().setStr(valueStr).build();
             case DECIMAL:
+                if (literal.getType().getPrecision() == IrDataTypeConvertor.UINT32_PRECISION
+                        && literal.getType().getScale() == IrDataTypeConvertor.UINT32_SCALE) {
+                    BigDecimal uint32Value = (BigDecimal) literal.getValue();
+                    return Common.Value.newBuilder().setU32(uint32Value.intValue()).build();
+                }
+                if (literal.getType().getPrecision() == IrDataTypeConvertor.UINT64_PRECISION
+                        && literal.getType().getScale() == IrDataTypeConvertor.UINT64_SCALE) {
+                    BigDecimal uint32Value = (BigDecimal) literal.getValue();
+                    return Common.Value.newBuilder().setU64(uint32Value.longValue()).build();
+                }
+                throw new UnsupportedOperationException(
+                        "decimal type with precision="
+                                + literal.getType().getPrecision()
+                                + ", scale="
+                                + literal.getType().getScale()
+                                + " is unsupported yet");
             case FLOAT:
+                return Common.Value.newBuilder()
+                        .setF32(((Number) literal.getValue()).floatValue())
+                        .build();
             case DOUBLE:
                 return Common.Value.newBuilder()
                         .setF64(((Number) literal.getValue()).doubleValue())

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphBuilder.java

@@ -42,6 +42,7 @@
 import com.alibaba.graphscope.common.ir.type.*;
 import com.alibaba.graphscope.gremlin.Utils;
 import com.alibaba.graphscope.proto.frontend.Code;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
@@ -96,6 +97,9 @@ protected GraphBuilder(Context context, GraphOptCluster cluster, RelOptSchema re
                 new GraphRexSimplify(
                         cluster.getRexBuilder(), RelOptPredicateList.EMPTY, RexUtil.EXECUTOR));
         this.configs = context.unwrapOrThrow(Configs.class);
+        if (FrontendConfig.DISABLE_EXPR_SIMPLIFY.get(this.configs)) {
+            disableSimplify();
+        }
     }
 
     /**
@@ -109,6 +113,12 @@ public static GraphBuilder create(
         return new GraphBuilder(context, cluster, relOptSchema);
     }
 
+    @VisibleForTesting
+    public void disableSimplify() {
+        Config config = Utils.getFieldValue(RelBuilder.class, this, "config");
+        config.withSimplify(false);
+    }
+
     public Context getContext() {
         return this.configs;
     }
@@ -1739,7 +1749,11 @@ public RexLiteral literal(@Nullable Object value) {
             return rexBuilder.makeLiteral((Boolean) value);
         } else if (value instanceof BigDecimal) {
             return rexBuilder.makeExactLiteral((BigDecimal) value);
-        } else if (value instanceof Float || value instanceof Double) {
+        } else if (value instanceof Float) {
+            return rexBuilder.makeApproxLiteral(
+                    BigDecimal.valueOf(((Number) value).floatValue()),
+                    getTypeFactory().createSqlType(SqlTypeName.FLOAT));
+        } else if (value instanceof Double) {
             return rexBuilder.makeApproxLiteral(BigDecimal.valueOf(((Number) value).doubleValue()));
         } else if (value instanceof Long) { // convert long to BIGINT, i.e. 2l
             return rexBuilder.makeBigintLiteral(BigDecimal.valueOf(((Number) value).longValue()));