Chore/add loss column

flockoff/validator/chain.py

@@ -54,7 +54,6 @@ def retrieve_model_metadata(
         model_id = None
         try:
             model_id = ModelId.from_compressed_str(chain_str)
-            bt.logging.info(f"Successfully parsed model ID: {model_id}")
         except Exception as e:
             # If the metadata format is not correct on the chain then we return None.
             bt.logging.error(

flockoff/validator/database.py

@@ -39,6 +39,7 @@ def _init_db(self):
             )
             self._add_column_if_not_exists(c, 'miner_scores', 'namespace', 'TEXT')
             self._add_column_if_not_exists(c, 'miner_scores', 'revision', 'TEXT')
+            self._add_column_if_not_exists(c, 'miner_scores', 'raw_loss', 'REAL')
 
             self.conn.commit()
         except sqlite3.Error as e:
@@ -154,6 +155,21 @@ def insert_or_reset_uid(
             logger.error(f"Failed to insert/reset UID {uid}: {str(e)}")
             raise DatabaseError(f"Failed to insert/reset UID: {str(e)}") from e
 
+    def update_raw_loss(self, uid: int, loss: float):
+        """Update the raw loss for a given UID."""
+        try:
+            c = self.conn.cursor()
+            c.execute(
+                "UPDATE miner_scores SET raw_loss = ? WHERE uid = ?", (loss, uid)
+            )
+            if c.rowcount == 0:
+                # If somehow a UID is being updated that wasn't inserted, log a warning or error.
+                logger.warning(f"Attempted to update raw_loss for non-existent UID {uid}, no changes made.")
+            self.conn.commit()
+        except sqlite3.Error as e:
+            logger.error(f"Failed to update raw_loss for UID {uid}: {str(e)}")
+            raise DatabaseError(f"Failed to update raw_loss: {str(e)}") from e
+
     def update_raw_eval_score(self, uid: int, new_raw_score: float):
         """Update the raw evaluation score for a given UID."""
         try:

flockoff/validator/trainer.py

@@ -57,9 +57,6 @@ def download_dataset(
     # only skip if we've recorded the same revision *and* dir still exists
     if last == revision and os.path.isdir(local_dir):
         if not force:
-            bt.logging.info(
-                f"[HF] {namespace}@{revision} already present; skipping download."
-            )
             return
     # if revision changed and dir exists, clear it so we'll redownload clean
     if last is not None and last != revision and os.path.isdir(local_dir):
@@ -81,6 +78,21 @@ def download_dataset(
     db.set_revision(namespace, revision)
     time.sleep(1)
 
+
+def check_valid_revision(namespace: str, revision: str):
+    try:
+        repo_info = HfApi(token=os.environ["HF_TOKEN"]).repo_info(repo_id=namespace, revision=revision, repo_type="dataset")
+    except Exception as e:
+        bt.logging.error(f"Error fetching repo info for repo {namespace} and revision {revision}: {e}")
+        return False
+    # Cut down the commit hash to the same amount of characters as the revision to compare them
+    # Enforce a 7 character length minimum for the revision to prevent collisions
+    revision_length = max(len(revision), 7)
+    if repo_info.sha[:revision_length] != revision:
+        bt.logging.error(f"revision {revision} does not match the commit hash {repo_info.sha}")
+        return False
+    return True
+
 def reset_gpu():
     """Reset GPU state and clear memory"""
     if torch.cuda.is_available():
@@ -119,7 +131,7 @@ def train_lora(
     try:
         # Reset GPU state at the start
         reset_gpu()
-        
+
         if cache_dir:
             os.makedirs(cache_dir, exist_ok=True)
             os.environ["HF_HOME"] = cache_dir
@@ -270,19 +282,19 @@ def train_lora(
 
         # Eval model
         eval_result = eval_trainer.evaluate()
-        
+
         # Thorough cleanup
         safe_cuda_cleanup(eval_model)
         safe_cuda_cleanup(eval_trainer)
         safe_cuda_cleanup(trainer)
         safe_cuda_cleanup(model)
         safe_cuda_cleanup(tokenizer)
-        
+
         # Clear any remaining CUDA memory
         reset_gpu()
-        
+
         return eval_result["eval_loss"]
-        
+
     except Exception as e:
         bt.logging.error(f"Error during training: {e}")
         # Attempt to clean up in case of error
@@ -300,5 +312,5 @@ def train_lora(
             reset_gpu()
         except Exception as cleanup_error:
             bt.logging.error(f"Error during cleanup after training error: {cleanup_error}")
-        
+
         return benchmark_loss

neurons/validator.py

@@ -38,6 +38,7 @@
 from flockoff.validator.trainer import (
     train_lora,
     download_dataset,
+    check_valid_revision
 )
 from flockoff.validator.database import ScoreDB
 
@@ -165,6 +166,29 @@ def __init__(self):
         self._update_score_init()
         bt.logging.info("Validator ready to run")
 
+    def get_registration_block(self, uid: int) -> typing.Optional[int]:
+        """Get the block at which a UID was registered on the subnet.
+
+        Args:
+            uid: The unique identifier of the neuron.
+
+        Returns:
+            The block number when the UID was registered, or None if query fails.
+        """
+        try:
+            result = self.subtensor.query_subtensor(
+                "BlockAtRegistration", 
+                params=[self.config.netuid, uid]
+            )
+            if result is not None:
+                # The result is a BittensorScaleType, extract the value
+                registration_block = int(result.value) if hasattr(result, 'value') else int(result)
+                return registration_block
+            return None
+        except Exception as e:
+            bt.logging.warning(f"Failed to get registration block for UID {uid}: {e}")
+            return None
+
     def _update_score_init(self):
         bt.logging.info("start to update score init")
         current_uids = self.metagraph.uids.tolist()
@@ -308,16 +332,30 @@ async def run_step(self):
 
             if metadata_i is None:
                 bt.logging.debug(
-                    f"Skipping UID {uid_i}  (metadata is None)"
+                    f"UID {uid_i} has no metadata, assigning default score"
                 )
+                raw_scores_this_epoch[uid_i] = constants.DEFAULT_RAW_SCORE
+                self.score_db.update_raw_eval_score(uid_i, constants.DEFAULT_RAW_SCORE)
                 continue
+
+            # Check if commitment block is greater than registration block
+            registration_block = self.get_registration_block(uid_i)
+
             metadata_per_uid[uid_i] = metadata_i  # Store metadata for this UID
             block_per_uid[uid_i] = metadata_i.block
 
             bt.logging.info(
-                f"Downloading training dataset: {metadata_i.id.namespace}/{metadata_i.id.commit}"
+                f"Downloading {uid_i}:{self.metagraph.hotkeys[uid_i]} training dataset: {metadata_i.id.namespace}/{metadata_i.id.commit}, block:{metadata_i.block}"
             )
             miner_i_data_dir = os.path.join(self.config.data_dir, f"miner_{uid_i}")
+            if registration_block is not None and metadata_i.block <= registration_block:
+                bt.logging.warning(
+                    f"UID {uid_i} has commitment block {metadata_i.block} <= registration block {registration_block}. "
+                    f"Assigning score of 0 to prevent claiming prior submissions."
+                )
+                raw_scores_this_epoch[uid_i] = constants.DEFAULT_RAW_SCORE
+                self.score_db.update_raw_eval_score(uid_i, constants.DEFAULT_RAW_SCORE)
+                continue
             download_dataset(
                 metadata_i.id.namespace,
                 metadata_i.id.commit,
@@ -362,7 +400,6 @@ async def run_step(self):
                 )
                 continue
 
-
             for uid_j in uids_to_eval:
                 if (
                         uid_i != uid_j
@@ -380,9 +417,6 @@ async def run_step(self):
                         continue
                     try:
                         os.makedirs(miner_j_data_dir, exist_ok=True)
-                        bt.logging.info(
-                            f"Downloading training dataset: {metadata_j.id.namespace}/{metadata_j.id.commit}"
-                        )
                         download_dataset(
                             metadata_j.id.namespace,
                             metadata_j.id.commit,
@@ -466,6 +500,13 @@ async def run_step(self):
                 revision = metadata.id.commit
                 last_rev = self.score_db.get_score_revision(uid, ns)
                 bt.logging.info(f"Metadata namespace: {ns}, commit: {revision}")
+                if not check_valid_revision(namespace=ns, revision=revision):
+                    raw_scores_this_epoch[uid] = constants.DEFAULT_RAW_SCORE
+                    self.score_db.update_raw_eval_score(uid, constants.DEFAULT_RAW_SCORE)
+                    bt.logging.info(
+                        f"Assigned fallback score {constants.DEFAULT_RAW_SCORE:.6f} to UID {uid} due to the dataset hash is invalid"
+                    )
+                    continue
                 if last_rev == revision:
                     bt.logging.info(
                         f"Skipping UID {uid} as it has already been evaluated with revision {revision}"
@@ -501,6 +542,7 @@ async def run_step(self):
 
                     raw_scores_this_epoch[uid] = eval_loss
                     self.score_db.update_raw_eval_score(uid, eval_loss)
+                    self.score_db.update_raw_loss(uid, eval_loss)
                     self.score_db.set_score_revision(uid, ns, revision, self.metagraph.hotkeys[uid])
 
                     bt.logging.info(f"Stored evaluation results for UID {uid}")

tests/FlockDataset/validators/test_database.py

@@ -18,7 +18,7 @@ def test_init_db(db):
     assert c.fetchone() is not None, "The 'miner_scores' table should be created"
     c.execute("PRAGMA table_info(miner_scores)")
     columns = {row[1] for row in c.fetchall()}
-    expected_columns = {"uid", "hotkey", "raw_score", "normalized_score", "namespace", "revision"}
+    expected_columns = {"uid", "hotkey", "raw_score", "normalized_score", "namespace", "revision", "raw_loss"}
     assert columns == expected_columns, f"Expected columns {expected_columns}, got {columns}"