Improve NameRes memory and Solr cache configuration

helm/name-lookup/Chart.yaml

@@ -14,7 +14,7 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.5.2
+version: 0.5.4
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.

helm/name-lookup/templates/scripts-config-map.yaml

@@ -73,6 +73,20 @@ data:
         -O- ${SOLR_SERVER}/solr/${COLLECTION_NAME}/config
     sleep 1
 
+    # Tune the queryResultCache. The _default configset ships with a 512-entry
+    # cache that is badly undersized for NameRes query traffic; see
+    # https://github.com/NCATSTranslator/NameResolution/issues/266
+    wget --post-data '{
+              "set-property": {
+                  "query.queryResultCache.size": {{ .Values.solr.queryResultCache.size }},
+                  "query.queryResultCache.initialSize": {{ .Values.solr.queryResultCache.initialSize }},
+                  "query.queryResultCache.maxRamMB": {{ .Values.solr.queryResultCache.maxRamMB }},
+                  "query.queryResultCache.autowarmCount": {{ .Values.solr.queryResultCache.autowarmCount }}
+              }}' \
+        --header='Content-Type:application/json' \
+        -O- ${SOLR_SERVER}/solr/${COLLECTION_NAME}/config
+    sleep 1
+
     # Restore data
     BACKUP_NAME="backup"
     CORE_NAME=${COLLECTION_NAME}_shard1_replica_n1

helm/name-lookup/values.yaml

@@ -61,16 +61,42 @@ solr:
       cpu: 4000m
     limits:
       memory: "32Gi"
-      cpu: 6000m
+      cpu: 7000m
 
   # You can control the nodeSelector/affinity/tolerations settings for Solr with the following settings.
   # Other pods (web, restore, backup) are controlled via app.nodeSelector/affinity/tolerations below.
   nodeSelector:
   affinity:
   tolerations:
 
-  heap_mem: "-Xms30G -Xmx30G"
-  gc: "-XX:NewSize=4G -XX:MaxNewSize=4G -XX:+UseG1GC -XX:MaxGCPauseMillis=1000 -XX:+UnlockExperimentalVMOptions -XX:G1MaxNewSizePercent=40 -XX:G1NewSizePercent=5 -XX:G1HeapRegionSize=32M -XX:InitiatingHeapOccupancyPercent=90"
+  # -Xms == -Xmx so the heap is allocated up front and never resized at runtime.
+  # 12G leaves ~20G of a 32Gi container for the OS page cache, which serves the
+  # mmap'd ~142G index (the live heap set is only a few GB; see #265/#267).
+  heap_mem: "-Xms12G -Xmx12G"
+
+  # G1 GC. We deliberately do NOT pin the young generation (no -XX:NewSize/
+  # MaxNewSize): a fixed young gen overrides G1NewSizePercent/G1MaxNewSizePercent
+  # and disables the MaxGCPauseMillis pause-time goal. Instead we let G1 size the
+  # young gen adaptively against a 200ms pause target, and let IHOP stay adaptive
+  # (no -XX:InitiatingHeapOccupancyPercent override). See
+  # https://github.com/NCATSTranslator/NameResolution/issues/272
+  gc: "-XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+UnlockExperimentalVMOptions -XX:G1MaxNewSizePercent=40 -XX:G1NewSizePercent=5 -XX:G1HeapRegionSize=32M"
+
+  # Solr query cache tuning, applied to the name_lookup collection via the Config
+  # API during the restore job (written to the collection's configoverlay.json).
+  #
+  # The Solr _default configset ships with a 512-entry queryResultCache, which
+  # was found to be badly undersized in production (~56% hit rate, 317K evictions)
+  # while the filterCache was performing well (99.8% hit ratio, no evictions).
+  # See https://github.com/NCATSTranslator/NameResolution/issues/266
+  #
+  # Solr 9.x already defaults to solr.search.CaffeineCache, so we only need to
+  # tune the sizing here. maxRamMB bounds memory regardless of entry count.
+  queryResultCache:
+    size: 4096
+    initialSize: 2048
+    maxRamMB: 512
+    autowarmCount: 0
 
   # As of Babel 2023jul13, we need 130G to store the
   # uncompressed backup + 131G Solr database