Speed up and simplify Solr data loading; fix /status for standalone

.github/workflows/tester.yml

@@ -22,8 +22,14 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Run SOLR
-        run: docker run --name name_lookup -d -p 8983:8983 -t solr -DzkRun
+      - name: Run SOLR (standalone)
+        run: docker run --name name_lookup -d -p 8983:8983 solr:9.10.0
+
+      - name: Create the name_lookup core from the checked-in configset
+        run: |
+          until curl -sf http://localhost:8983/solr/admin/cores?action=STATUS >/dev/null; do sleep 2; done
+          docker cp data-loading/configsets/name_lookup name_lookup:/tmp/name_lookup
+          docker exec name_lookup solr create -c name_lookup -d /tmp/name_lookup
 
       - name: Install dependencies
         run: |

CLAUDE.md

@@ -10,7 +10,12 @@ NameRes (Name Resolver) is a biomedical entity name resolution service that maps
 
 ### Running Tests
 ```bash
-# Load test data into Solr first (requires Solr running)
+# Start standalone Solr and create the name_lookup core from the checked-in configset
+docker run --name name_lookup -d -p 8983:8983 solr:9.10.0
+docker cp data-loading/configsets/name_lookup name_lookup:/tmp/name_lookup
+docker exec name_lookup solr create -c name_lookup -d /tmp/name_lookup
+
+# Load test data into the core (parallel load, with a document-count guard)
 ./data-loading/setup-and-load-solr.sh tests/data/test-synonyms.json
 
 # Run all tests

data-loading/Dockerfile

@@ -67,12 +67,14 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 COPY --chown=nru requirements.txt ${ROOT}
 RUN pip3 install -r requirements.txt
 
-# Copy necessary files.
+# Copy necessary files. The configset carries the Solr schema/config; the core is
+# created from it with `solr create -c name_lookup -d configsets/name_lookup`.
 COPY --chown=nru setup-and-load-solr.sh ${ROOT}
 COPY --chown=nru README.md ${ROOT}
 COPY --chown=nru Makefile ${ROOT}
+COPY --chown=nru configsets ${ROOT}/configsets
 
-# On entry, start the Solr instance.
+# On entry, start the Solr instance (standalone -- no ZooKeeper for a one-node load).
 ENV SOLR_EXEC="${ROOT}/solr/solr-${SOLR_VERSION}/bin/solr"
 ENV SOLR_DIR="$SOLR_DIR"
-ENTRYPOINT ${SOLR_EXEC} -cloud -f -p 8983 -m 64G -s ${SOLR_DIR}
+ENTRYPOINT ${SOLR_EXEC} start -f --user-managed -p 8983 -m 64G -s ${SOLR_DIR}

data-loading/Makefile

@@ -1,83 +1,95 @@
 # This Makefile contains all the instructions necessary to
-# download Babel synonym files from a web location, create
-# and load them into a Solr dataset and generate a Solr backup
-# that can be used to start a NameRes instance.
+# download Babel synonym files from a web location, load them into a
+# standalone Solr core (using the checked-in configset under
+# configsets/name_lookup) and generate a self-contained Solr backup that can be
+# used to start a NameRes instance.
 #
+# The backup is the whole Solr core -- config, schema AND index -- so restoring
+# it is just "untar into the Solr home and start Solr" (see ../data-loading/README.md).
 
 # Configuration
 SYNONYMS_URL=https://stars.renci.org/var/babel_outputs/2025sep1/synonyms/
 
 # How much memory should Solr use.
 SOLR_MEM=220G
 
-# SOLR_DIR should be set up to point to the Solr data directory (usually /var/solr)
-# and SOLR_EXEC should be set up to point to the Solr executable.
-# These will both be set up by the Dockerfile.
+# The checked-in configset used to create the core (contains conf/).
+CONFIGSET=configsets/name_lookup
+
+# The Solr core name. Do not change without also changing NameRes and the configset.
+CORE=name_lookup
+
+# Split any synonym file larger than SPLIT_SIZE into chunks of SPLIT_LINES lines.
+# Splitting only helps parallelism, so we only bother with the big files; JSON is
+# one document per line, so we must split on line boundaries, never on bytes.
+SPLIT_SIZE=2G
+SPLIT_LINES=10000000
+
+# SOLR_DIR should be set up to point to the Solr data directory (usually /var/solr,
+# which is also the Solr home) and SOLR_EXEC should be set up to point to the Solr
+# executable. These will both be set up by the Dockerfile.
 
 # All and clean targets.
 
 .PHONY: all clean
 all: data/setup.done
 	echo Solr has now been set up and loaded with the synonym data.
-	echo Run 'make start-solr-backup' to start a backup. Run 'make check-solr-backup' to check
-	echo if the backup has completed. Once that has completed, run 'make data/backup.done' to
+	echo Run 'make data/backup.done' to optimize the index, shut down Solr and
 	echo generate a snapshot.backup.tar.gz file that can be used in NameRes.
 
 clean:
 	rm -rf data/*
 	mkdir data
 
-# This is a three step process.
-#
-# Step 1. Download an uncompress synonym files.
+# Step 1. Download and uncompress synonym files, then split the large ones so the
+# parallel loader has more units of work.
 data/synonyms/done:
 	mkdir -p data/synonyms
 	wget -c -r -l1 -nd -P data/synonyms ${SYNONYMS_URL}
 	gunzip data/synonyms/*.txt.gz
 	echo Downloaded synonyms from ${SYNONYMS_URL}
-	# split -d -l 10000000 data/synonyms/SmallMolecule.txt data/synonyms/SmallMolecule.txt. && rm data/synonyms/SmallMolecule.txt
-	split -d -l 10000000 data/synonyms/DrugChemicalConflated.txt data/synonyms/DrugChemicalConflated.txt. && rm data/synonyms/DrugChemicalConflated.txt
-	split -d -l 10000000 data/synonyms/GeneProteinConflated.txt data/synonyms/GeneProteinConflated.txt. && rm data/synonyms/GeneProteinConflated.txt
-	echo Split DrugChemicalConflated.txt and GeneProteinConflated.txt, and deleted the original files.
+	find data/synonyms -maxdepth 1 -type f -name '*.txt' -size +${SPLIT_SIZE} -print | while read -r f; do \
+		echo "Splitting large file $$f (larger than ${SPLIT_SIZE})..."; \
+		split -d -l ${SPLIT_LINES} "$$f" "$$f." && rm "$$f"; \
+	done
 	touch $@
 
-# Step 3. Start Solr server.
+# Step 2. Start the Solr server (standalone -- no ZooKeeper needed for one node).
+# --user-managed keeps standalone mode explicit (Solr 10 will default to cloud).
 data/solr.pid:
 	mkdir -p ${SOLR_DIR}/logs
-	${SOLR_EXEC} -cloud -p 8983 -v -m ${SOLR_MEM} -s ${SOLR_DIR} >> ${SOLR_DIR}/logs/solr.txt 2>> ${SOLR_DIR}/logs/solr.err.txt
+	${SOLR_EXEC} start -p 8983 --user-managed -v -m ${SOLR_MEM} -s ${SOLR_DIR} >> ${SOLR_DIR}/logs/solr.txt 2>> ${SOLR_DIR}/logs/solr.err.txt
 	while [ ! -s $@ ]; do \
 		${SOLR_EXEC} status | grep -Po 'Solr process \K([0-9]+)' > $@; \
 	done
 	$(info Solr started with PID file at $@)
 	cat $@
 
-# Step 4. Load JSON files into Solr server.
-data/setup.done: data/synonyms/done data/solr.pid
+# Step 3. Create the core from the checked-in configset (schema lives there).
+data/core.done: data/solr.pid
+	${SOLR_EXEC} create -c ${CORE} -d ${CONFIGSET}
+	touch $@
+
+# Step 4. Load the synonym files into the core (parallel; with a doc-count guard).
+data/setup.done: data/synonyms/done data/core.done
 	mkdir -p data/logs
 	bash setup-and-load-solr.sh "data/synonyms/*.txt*" >> data/logs/setup-and-load-solr.sh.log 2>> data/logs/setup-and-load-solr.sh.err.log && touch $@
 
-# Step 5. Start a Solr backup.
-.PHONY: start-solr-backup
-start-solr-backup: data/setup.done
-	curl 'http://localhost:8983/solr/name_lookup/replication?command=backup&name=backup'
-
-# Step 6. Wait for the backup to complete.
-.PHONY: check-solr-backup
-check-solr-backup:
-	curl 'http://localhost:8983/solr/name_lookup/replication?command=details'
-
-# Step 6. Shutdown the Solr instance.
-### data/stop-solr:
-###    	docker exec name_lookup solr stop -p 8983 -verbose
-
-# Step 7. Generate the backup tarball.
-data/backup.done:
-	mkdir -p data/var/solr/data
-	mv /var/solr/name_lookup_shard1_replica_n1/data/snapshot.backup data/var/solr/data
-	cd data && tar zcvf snapshot.backup.tar.gz var && touch backup.done
+# Step 5. Optimize the index, shut down Solr, and tar the whole core into a
+# self-contained backup. Extracting snapshot.backup.tar.gz into the Solr home
+# yields a ready-to-serve name_lookup/ core -- no restore-time schema setup.
+data/backup.done: data/setup.done
+	@echo "Optimizing the index before export (this can take a while on a large index)..."
+	curl -sf --show-error 'http://localhost:8983/solr/${CORE}/update?optimize=true'
+	# The optimize above commits, so the on-disk index is consistent. `solr stop`
+	# can exit non-zero under Docker even on a clean stop (SOLR-16463), so ignore it.
+	${SOLR_EXEC} stop -p 8983 || true
+	rm -f data/solr.pid
+	tar -C ${SOLR_DIR} -czvf data/snapshot.backup.tar.gz ${CORE}
+	touch $@
 
 .PHONY: stop-solr
 stop-solr:
-	rm data/solr.pid
-	${SOLR_EXEC} stop
+	rm -f data/solr.pid
+	${SOLR_EXEC} stop -p 8983 || true
 	$(info Solr stopped.)