From ca97f90861989b216e865c79bd15e4824b7bfe36 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 2 Jun 2026 13:35:18 -0400 Subject: [PATCH 1/5] Reduced NameRes heap_mem, removed minimum memory. As per https://github.com/NCATSTranslator/NameResolution/issues/265 --- helm/name-lookup/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/name-lookup/values.yaml b/helm/name-lookup/values.yaml index 50fd18d4..41b08ccd 100644 --- a/helm/name-lookup/values.yaml +++ b/helm/name-lookup/values.yaml @@ -69,7 +69,7 @@ solr: affinity: tolerations: - heap_mem: "-Xms30G -Xmx30G" + heap_mem: "-Xmx12G" gc: "-XX:NewSize=4G -XX:MaxNewSize=4G -XX:+UseG1GC -XX:MaxGCPauseMillis=1000 -XX:+UnlockExperimentalVMOptions -XX:G1MaxNewSizePercent=40 -XX:G1NewSizePercent=5 -XX:G1HeapRegionSize=32M -XX:InitiatingHeapOccupancyPercent=90" # As of Babel 2023jul13, we need 130G to store the From c07a59c3bcbd712d91f0f4ee5928484caf3d1951 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 2 Jun 2026 14:00:50 -0400 Subject: [PATCH 2/5] Enlarge Solr queryResultCache for NameRes. The _default configset's 512-entry queryResultCache was badly undersized (~56% hit rate, 317K evictions) while filterCache was healthy. Tune it via the Config API during the restore job, configurable through values.yaml. See https://github.com/NCATSTranslator/NameResolution/issues/266 Co-Authored-By: Claude Opus 4.8 --- helm/name-lookup/Chart.yaml | 2 +- .../templates/scripts-config-map.yaml | 14 ++++++++++++++ helm/name-lookup/values.yaml | 16 ++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/helm/name-lookup/Chart.yaml b/helm/name-lookup/Chart.yaml index bd5ada54..a99e8145 100644 --- a/helm/name-lookup/Chart.yaml +++ b/helm/name-lookup/Chart.yaml @@ -14,7 +14,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.5.2 +version: 0.5.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. diff --git a/helm/name-lookup/templates/scripts-config-map.yaml b/helm/name-lookup/templates/scripts-config-map.yaml index 189c7372..4ac8b837 100644 --- a/helm/name-lookup/templates/scripts-config-map.yaml +++ b/helm/name-lookup/templates/scripts-config-map.yaml @@ -73,6 +73,20 @@ data: -O- ${SOLR_SERVER}/solr/${COLLECTION_NAME}/config sleep 1 + # Tune the queryResultCache. The _default configset ships with a 512-entry + # cache that is badly undersized for NameRes query traffic; see + # https://github.com/NCATSTranslator/NameResolution/issues/266 + wget --post-data '{ + "set-property": { + "query.queryResultCache.size": {{ .Values.solr.queryResultCache.size }}, + "query.queryResultCache.initialSize": {{ .Values.solr.queryResultCache.initialSize }}, + "query.queryResultCache.maxRamMB": {{ .Values.solr.queryResultCache.maxRamMB }}, + "query.queryResultCache.autowarmCount": {{ .Values.solr.queryResultCache.autowarmCount }} + }}' \ + --header='Content-Type:application/json' \ + -O- ${SOLR_SERVER}/solr/${COLLECTION_NAME}/config + sleep 1 + # Restore data BACKUP_NAME="backup" CORE_NAME=${COLLECTION_NAME}_shard1_replica_n1 diff --git a/helm/name-lookup/values.yaml b/helm/name-lookup/values.yaml index 41b08ccd..e5623ab9 100644 --- a/helm/name-lookup/values.yaml +++ b/helm/name-lookup/values.yaml @@ -72,6 +72,22 @@ solr: heap_mem: "-Xmx12G" gc: "-XX:NewSize=4G -XX:MaxNewSize=4G -XX:+UseG1GC -XX:MaxGCPauseMillis=1000 -XX:+UnlockExperimentalVMOptions -XX:G1MaxNewSizePercent=40 -XX:G1NewSizePercent=5 -XX:G1HeapRegionSize=32M -XX:InitiatingHeapOccupancyPercent=90" + # Solr query cache tuning, applied to the name_lookup collection via the Config + # API during the restore job (written to the collection's configoverlay.json). + # + # The Solr _default configset ships with a 512-entry queryResultCache, which + # was found to be badly undersized in production (~56% hit rate, 317K evictions) + # while the filterCache was performing well (99.8% hit ratio, no evictions). + # See https://github.com/NCATSTranslator/NameResolution/issues/266 + # + # Solr 9.x already defaults to solr.search.CaffeineCache, so we only need to + # tune the sizing here. maxRamMB bounds memory regardless of entry count. + queryResultCache: + size: 4096 + initialSize: 2048 + maxRamMB: 512 + autowarmCount: 0 + # As of Babel 2023jul13, we need 130G to store the # uncompressed backup + 131G Solr database # So 300Gi should be enough for the time being. From dcc6d73613090736049fedb621b4ecdb918d5bfd Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 2 Jun 2026 14:14:37 -0400 Subject: [PATCH 3/5] Clean up contradictory Solr G1 GC flags. The young generation was pinned (-XX:NewSize/MaxNewSize=4G), which overrode the adaptive G1NewSizePercent bounds and disabled the MaxGCPauseMillis goal. Drop the pin so G1 sizes young gen against a 200ms pause target, let IHOP stay adaptive, and set -Xms == -Xmx to avoid runtime heap resizing. See https://github.com/NCATSTranslator/NameResolution/issues/272 Co-Authored-By: Claude Opus 4.8 --- helm/name-lookup/Chart.yaml | 2 +- helm/name-lookup/values.yaml | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/helm/name-lookup/Chart.yaml b/helm/name-lookup/Chart.yaml index a99e8145..74137c8c 100644 --- a/helm/name-lookup/Chart.yaml +++ b/helm/name-lookup/Chart.yaml @@ -14,7 +14,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.5.3 +version: 0.5.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. diff --git a/helm/name-lookup/values.yaml b/helm/name-lookup/values.yaml index e5623ab9..e745bd3c 100644 --- a/helm/name-lookup/values.yaml +++ b/helm/name-lookup/values.yaml @@ -69,8 +69,18 @@ solr: affinity: tolerations: - heap_mem: "-Xmx12G" - gc: "-XX:NewSize=4G -XX:MaxNewSize=4G -XX:+UseG1GC -XX:MaxGCPauseMillis=1000 -XX:+UnlockExperimentalVMOptions -XX:G1MaxNewSizePercent=40 -XX:G1NewSizePercent=5 -XX:G1HeapRegionSize=32M -XX:InitiatingHeapOccupancyPercent=90" + # -Xms == -Xmx so the heap is allocated up front and never resized at runtime. + # 12G leaves ~20G of a 32Gi container for the OS page cache, which serves the + # mmap'd ~142G index (the live heap set is only a few GB; see #265/#267). + heap_mem: "-Xms12G -Xmx12G" + + # G1 GC. We deliberately do NOT pin the young generation (no -XX:NewSize/ + # MaxNewSize): a fixed young gen overrides G1NewSizePercent/G1MaxNewSizePercent + # and disables the MaxGCPauseMillis pause-time goal. Instead we let G1 size the + # young gen adaptively against a 200ms pause target, and let IHOP stay adaptive + # (no -XX:InitiatingHeapOccupancyPercent override). See + # https://github.com/NCATSTranslator/NameResolution/issues/272 + gc: "-XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+UnlockExperimentalVMOptions -XX:G1MaxNewSizePercent=40 -XX:G1NewSizePercent=5 -XX:G1HeapRegionSize=32M" # Solr query cache tuning, applied to the name_lookup collection via the Config # API during the restore job (written to the collection's configoverlay.json). From 6ab996eaada2243bb8cbd5ca620153c901509a98 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 2 Jun 2026 15:57:56 -0400 Subject: [PATCH 4/5] Increased the cpu limit slightly (6 -> 7). --- helm/name-lookup/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/name-lookup/values.yaml b/helm/name-lookup/values.yaml index e745bd3c..64b427dc 100644 --- a/helm/name-lookup/values.yaml +++ b/helm/name-lookup/values.yaml @@ -61,7 +61,7 @@ solr: cpu: 4000m limits: memory: "32Gi" - cpu: 6000m + cpu: 7000m # You can control the nodeSelector/affinity/tolerations settings for Solr with the following settings. # Other pods (web, restore, backup) are controlled via app.nodeSelector/affinity/tolerations below. From 6bb7ea02625612c68a2c90eb97bff297bd8d24c1 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 2 Jun 2026 16:55:59 -0400 Subject: [PATCH 5/5] Increased memory and CPU for NameRes Dev at RENCI. --- .../renci-dev-values-populated.yaml | Bin 930 -> 1187 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/helm/name-lookup/renci-dev-values-populated.yaml b/helm/name-lookup/renci-dev-values-populated.yaml index 9686e14d015f8fb337a93eba75f0acb711d1b069..b7e2656b5dcdb36db3e3bebadbcc9069be07555e 100644 GIT binary patch literal 1187 zcmV;U1YG+7M@dveQdv+`094M4Sb{_RW)L7KT5+D`(A}Dgw~7jVfF#u5dz;3P9pIKN z2muV)SuD1V%?cf5t|4FH%#9^KGOo47^X3uh47o{jDts&%f+{B@tsUx!jHW4d;KKsNX&UllM8kD%`>@LMXYf@&o< zV&g~iAUru|fUIKs0w{DUyMgqh z1`K{0_ZCQxA$0KZdc78;1M)LTVnM_RXcYSq(wLaUoWC6Mqgju`Vqx8Vbi(h;2Wq3M zx}|ou9(SJ5Xo`!^O7rDK1tv=R@=P#5VJ{gtDcU)D#KfApGu1yd&{e5tpvZy4Usz>N zeaM`04={QN!xz#dj)UTMT(shE1Gxrh94l{fE)*cR3K{V9=%mi8lyaUd-+1x;%wcpA z9cE_l@7MU5p*tbGXYch4n~>+1DQz4!e@rmP@&H3E$&i2sa_^S$BfT@>6BgF9*~FsI zl!Jenlc=$9OW(eM$B_1K<45f?#tAc=u@8=|q%IA)HhtnKp|#9VcCZv!1djL?t{Oqi zCq+Hd-%sT3s^}U&A32#>eMp6wQF7Lm|JQa^ya?@HQ%oD*%Lvs2zABqA0HkqtnL)v1 zD$xKioc3tvR=G=Q!T%Wi_-(6`7hU2~HRv}$pYLpX(2KMRePnQqs#54o{74F)H0SrAEXbs!-YoiCM?XjMqgh)dxd_ z&vk-GSI^G?tgJr^WNqtNxsque^zNNBVGPWPpocNh&aaUFCk$S+bZJhfCWMG7h&(G? zUyI5FzQHe626gM#N{%~=K2UObYin`EkR+8aRt-@+9#e{ZM1zU}w(I7%iC-*iaUtm) z=-yunbqdA8!$5NJ6_1Z|GS}aB)*LaGNidd}Gu#o_C0yX=A~K9JJyyR3fs;pBy%#4( z)fC|r1YekUm_A%fjrW?-SWq37tOw=JB^Tix?0BCKd>YWmC76P6nC=Oku*akwgsg+E=BaJvE6$~WSnIs zQ_ literal 930 zcmV;T16}+8M@dveQdv+`0NV^CPT9Q|QX125u3v}it}s^wQz_@5;cMP-S%D(fY+OP$ zjxi+W84dEGWKH3J*Cn|m=hFr;HCF0I9I^?de+Vw`t6;Jc_y%r^0A;?ov%q~1BuQPQ zH!>S4Xhzy-%`zi?rvMLOyV4vl=h6M8qGZd6DESoJcmd*Btc5IS@xP0om=+EDcR1v9 zP0WMnzR;INVmSUDq+pU$@D~QoN{{J|sV~;Amsq+MH-vg|$=Hr8f0FdS61W#&WMYkr z5Cs$NIB=5|id)hJm)$cm2|L)W10clAo6?bgd-4F+Gw2vb48f&47wjbtQxpiDdKa5+{YK7YXSILo_Xf~6>z3`x91oVtz90t+>MSE? zAuIg9s_R4l$5Svf0i@K{O)QZ{U4lLp~*~Z(fq>~?)K!?5zFkjY26DXAn%1!)!UO)gm4E=uQPS+ zGPCnfJt2znvy_L6t*=e2#j|N(Eq#_a7(Kml3e6d8H1^Br)*sR-Q+bS!wxyGju+zf@ zJ;!GXRiAEFAdO3n$}0A5Ze*>ifPwR|Bd96w7#O_F{km2tK<+mY^uV$AKGkr$DlNAC zz(wVVGSzh~)3&KhcRG*U91PNSS0%Psx#xPdYAzTFmvwZ1<=kS5X)*f8IesO(4%-pu zDh6Ri&gyaEBqPtFUAO!!5wYNURW`#T#2xft2g=o8MYJIFwpb8OEcy6?2;PfUalGL$ ze96Ia2&uwyIiqX3)iW-HSGRgpiQ4n&4}9T|Kv3(-KF>Ca7qgYn#IIF|*^S{0?N|Rl z?gzHwegCB@?1WH$^?vg~m@|XE4dR?*NeW+?C_`JajX&%mDQmV^6_}ALZN6P1`u-LaT#Mq-qo`UsB{`X{926ESat! E{7%Ema{vGU