[Feature] Improve (dis-)connect handling between peers

.ci/devnet_ci.sh

@@ -14,6 +14,7 @@ total_validators=$1
 total_clients=$2
 network_id=$3
 min_height=$4
+max_warnings=$5
 
 # The verobsity of snarkos nodes.
 NODE_VERBOSITY=3
@@ -23,6 +24,7 @@ NODE_VERBOSITY=3
 : "${total_clients:=4}" # need at least 4 clients, so each validator has at least one client connected to it.
 : "${network_id:=0}"
 : "${min_height:=60}" # To likely go past the 100 round garbage collection limit.
+: "${max_warnings:=10}"
 
 # shellcheck source=SCRIPTDIR/utils.sh
 . ./.ci/utils.sh
@@ -395,7 +397,7 @@ while (( total_wait < 600 )); do  # 10 minutes max
   if check_heights 0 $((total_validators+total_clients)) "$min_height" "$network_name" "$total_wait"; then
     log "🎉 Test passed! All nodes reached minimum height."
 
-    if check_logs "$log_dir" "$total_validators" "$total_clients"; then
+    if check_logs "$log_dir" "$total_validators" "$total_clients" "$max_warnings"; then
       exit 0
     else
       exit 1

.ci/utils.sh

@@ -116,7 +116,11 @@ function check_logs() {
   local log_dir=$1
   local total_validators=$2
   local total_clients=$3
-
+  # The maximum number of warnings allow in each node's log file.
+  # Nodes may create some warnings at startup because they cannot connect to each other yet.
+  local max_warnings=$4
+
+
   local all_reached=true
   local highest_height=0
 
@@ -132,6 +136,12 @@ function check_logs() {
       grep "ERROR" "$log_dir/validator-${validator_index}.log"
       return 1
     fi
+
+    num_warnings=$(grep -c "WARN" "$log_dir/validator-${validator_index}.log")
+    if (( num_warnings > max_warnings )); then
+      echo "❌ Test failed! Validator #${validator_index} logs contain more than ${max_warnings} warnings."
+      return 1
+    fi
   done
 
   for client_index in $(seq 0 $((total_clients-1))); do
@@ -146,7 +156,12 @@ function check_logs() {
       grep "ERROR" "$log_dir/client-${client_index}.log"
       return 1
     fi
-
+
+    num_warnings=$(grep -c "WARN" "$log_dir/client-${client_index}.log")
+    if (( num_warnings > max_warnings )); then
+      echo "❌ Test failed! Client #${client_index} logs contain more than ${max_warnings} warnings."
+      return 1
+    fi
   done
 
   return 0

.circleci/config.yml

@@ -420,6 +420,10 @@ commands:
       - run:
           name: "Install snarkos"
           no_output_timeout: 20m
+          environment:
+            CONSENSUS_VERSION_HEIGHTS: "0,100"
+            EXPECTED_MAX_CONSENSUS_VERSION: "10"
+            WAIT_BETWEEN_UPGRADES: "60"
           command: |
             cargo install --locked --path . --features test_network
       - run: