Release/redesign 1.0

flockoff/constants.py

@@ -1,5 +1,4 @@
 from pathlib import Path
-import bittensor as bt
 from dataclasses import dataclass
 from typing import Optional
 
@@ -8,22 +7,29 @@
 MIN_WEIGHT_THRESHOLD = 1e-6
 DEFAULT_RAW_SCORE = 999
 DEFAULT_NORMALIZED_SCORE = 0.0
-DEFAULT_DUPLICATE_COUNT = 100
+DEFAULT_DUPLICATE_COUNT = 33
 
 SCORE_PRECISION = 10_000
 
+SELECTION_SIZE = 100            # miners choose 100 rows
+LOSS_THRESHOLD_PCT = 0.05
+
+submission_start_utc_min = 12 * 60
+submission_window_mins = 30
+validate_start_utc_min = 12 * 60 + submission_window_mins
+reward_start_utc_min = 11 * 60 + 30
 
 @dataclass
 class Competition:
     """Class defining model parameters"""
     id: str = "1"
-    repo: str = "flock-io/flock-off-s1-character-roleplay"
+    repo: str = "flock-io/flock-off-s1-competition"
     bench: float = 2.60
     minb: float = 2.40
     maxb: float = 2.80
     bheight: float = 0.05
     pow: int = 2
-    rows: int = 250
+    rows: int = SELECTION_SIZE
 
     @classmethod
     def from_defaults(cls) -> "Competition":
@@ -32,4 +38,6 @@ def from_defaults(cls) -> "Competition":
 
 
 # eval dataset huggingface
-eval_commit = "784fbf1e78d16c512750e3bb5391fa6b338818ae"
+eval_commit = "main"
+
+

flockoff/miners/data.py

@@ -8,7 +8,7 @@
 # The length, in bytes, of a base64 encoded sha256 hash.
 SHA256_BASE_64_LENGTH = 44
 # The max length, in characters, of the competition id
-MAX_COMPETITION_ID_LENGTH = 2
+MAX_COMPETITION_ID_LENGTH = 12
 
 
 class ModelId(BaseModel):

flockoff/validator/trainer.py

@@ -25,6 +25,7 @@
 from .constants import model2template
 import bittensor as bt
 from flockoff.validator.database import ScoreDB
+from flockoff import constants
 
 api = HfApi()
 
@@ -40,7 +41,7 @@ class LoraTrainingArguments:
 
 
 def download_dataset(
-    namespace: str, revision: str, local_dir: str = "data", cache_dir: str = None, force: bool = False
+        namespace: str, revision: str, local_dir: str = "data", cache_dir: str = None, force: bool = False
 ):
     # Create cache directory if it doesn't exist
     if cache_dir:
@@ -52,7 +53,7 @@ def download_dataset(
         local_dir = os.path.abspath(local_dir)
 
     db = ScoreDB("scores.db")
-    last = db.get_revision(namespace)
+    last = db.get_revision(namespace, local_dir)
 
     # only skip if we've recorded the same revision *and* dir still exists
     if last == revision and os.path.isdir(local_dir):
@@ -71,12 +72,14 @@ def download_dataset(
     os.makedirs(local_dir, exist_ok=True)
 
     bt.logging.info(f"[HF] Downloading dataset {namespace}@{revision} → {local_dir}")
-    api.snapshot_download(
-        repo_id=namespace, local_dir=local_dir, revision=revision, repo_type="dataset"
-    )
-
-    db.set_revision(namespace, revision)
-    time.sleep(1)
+    try:
+        api.snapshot_download(
+            repo_id=namespace, local_dir=local_dir, revision=revision, repo_type="dataset"
+        )
+        db.set_revision(namespace, revision, local_dir)
+        time.sleep(1)
+    except Exception as e:
+        bt.logging.error(f"api.snapshot_download error:{e}")
 
 
 def check_valid_revision(namespace: str, revision: str):
@@ -93,6 +96,16 @@ def check_valid_revision(namespace: str, revision: str):
         return False
     return True
 
+
+def get_hg_revision(namespace: str, eval_commit: str):
+    try:
+        repo_info = HfApi(token=os.environ["HF_TOKEN"]).repo_info(repo_id=namespace, revision=eval_commit, repo_type="dataset")
+    except Exception as e:
+        bt.logging.error(f"Error fetching repo info for repo {namespace} and revision {eval_commit}: {e}")
+        return None
+    return repo_info.sha
+
+
 def reset_gpu():
     """Reset GPU state and clear memory"""
     if torch.cuda.is_available():
@@ -109,6 +122,7 @@ def reset_gpu():
         except Exception as e:
             bt.logging.error(f"Error resetting GPU: {e}")
 
+
 def safe_cuda_cleanup(model):
     """Safely move model to CPU and delete it"""
     try:
@@ -120,13 +134,14 @@ def safe_cuda_cleanup(model):
     finally:
         gc.collect()
 
+
 def train_lora(
-    lucky_num: int,
-    benchmark_loss: float,
-    eval_size: int,
-    cache_dir: str = None,
-    data_dir: str = "data",
-    eval_data_dir: str = "eval_data",
+        lucky_num: int,
+        benchmark_loss: float,
+        eval_size: int,
+        cache_dir: str = None,
+        data_dir: str = "data",
+        eval_data_dir: str = "eval_data",
 ) -> float:
     try:
         # Reset GPU state at the start
@@ -139,14 +154,7 @@ def train_lora(
 
         # set the same random seed to detect duplicate data sets
         from dotenv import load_dotenv
-
         load_dotenv()
-        os.environ["PYTHONHASHSEED"] = str(lucky_num)
-
-        torch.manual_seed(lucky_num)
-        if torch.cuda.is_available():
-            torch.cuda.manual_seed(lucky_num)
-            torch.cuda.manual_seed_all(lucky_num)
 
         CONTEXT_LENGTH = 2048
         with open(f"flockoff/validator/training_args.yaml", "r") as f:

flockoff/validator/validator_utils.py

@@ -1,18 +1,21 @@
 import json
+from typing import Tuple, Any
+
 import bittensor as bt
 import numpy as np
 from flockoff import constants
+from flockoff.validator.database import ScoreDB
 
 
 def compute_score(
-    loss,
-    benchmark_loss,
-    min_bench,
-    max_bench,
-    power,
-    bench_height,
-    miner_comp_id,
-    real_comp_id,
+        loss,
+        benchmark_loss,
+        min_bench,
+        max_bench,
+        power,
+        bench_height,
+        miner_comp_id,
+        real_comp_id,
 ):
     """
     Compute the score based on the loss and benchmark loss.
@@ -83,14 +86,73 @@ def compute_score(
         denominator = np.pow((max_bench - benchmark_loss), power)
         return numerator / denominator + bench_height
 
+
+def select_winner(db: ScoreDB, competition_id: str, hotkeys: dict, coldkeys: dict) -> tuple[None, None] | tuple[int, float]:
+    subs = db.get_competition_submissions(competition_id)
+    scored = [s for s in subs.values() if s.get('eval_loss') is not None]
+    if not scored:
+        return None, None
+
+    threshold_number = max(int(len(hotkeys) * constants.LOSS_THRESHOLD_PCT), 1)
+    scored_by_loss = sorted(scored, key=lambda s: s['eval_loss'])
+    eligible = scored_by_loss[:threshold_number]
+
+    if not eligible:
+        return None, None
+
+    def sort_key(s):
+        return (s.get('commitment_block', 10 ** 18), s.get('commitment_timestamp', 10 ** 18))
+
+    eligible_sorted = sorted(eligible, key=sort_key)
+    bt.logging.info(f"competition_id:{competition_id} , eligible_sorted:{eligible_sorted}")
+    winner = eligible_sorted[0]
+
+    uid = winner['uid']
+    if winner['hotkey'] != hotkeys[uid]:
+        replacement_found = False
+        for hotkey_uid, hotkey in hotkeys.items():
+            if hotkey == winner['hotkey']:
+                winner['uid'] = hotkey_uid
+                replacement_found = True
+            if replacement_found:
+                bt.logging.info(f"{competition_id} competition_id winner found in hotkeys :{hotkey_uid}")
+                return winner['uid'], winner['eval_loss']
+
+        for coldkey_uid, coldkey in coldkeys.items():
+            if coldkey == winner['coldkey']:
+                winner['uid'] = coldkey_uid
+                replacement_found = True
+            if replacement_found:
+                bt.logging.info(f"{competition_id} competition_id winner found in coldkeys :{coldkey_uid}")
+                return winner['uid'], winner['eval_loss']
+
+        if not replacement_found:
+            for candidate in eligible_sorted:
+                candidate_uid = candidate['uid']
+                if candidate_uid != uid and hotkeys[candidate_uid] == candidate['hotkey']:
+                    winner['uid'] = candidate_uid
+                    bt.logging.info(f"{competition_id} competition_id winner found in eligible_sorted :{candidate_uid}")
+                    return winner['uid'], winner['eval_loss']
+
+            for candidate in scored_by_loss:
+                candidate_uid = candidate['uid']
+                if candidate_uid != uid and hotkeys[candidate_uid] == candidate['hotkey']:
+                    winner['uid'] = candidate_uid
+                    bt.logging.info(f"{competition_id} competition_id winner found in scored_by_loss :{candidate_uid}")
+                    break
+
+    return winner['uid'], winner['eval_loss']
+
+
 def load_jsonl(path, max_rows=None):
     with open(path, 'r', encoding='utf-8') as f:
         data = [json.loads(line.strip()) for line in f if line.strip()]
         if max_rows is not None:
             data = data[:max_rows]
         return data
 
+
 def count_similar(jsonl1, jsonl2):
     set1 = set(json.dumps(item, sort_keys=True) for item in jsonl1)
     set2 = set(json.dumps(item, sort_keys=True) for item in jsonl2)
-    return len(set1 & set2)
+    return len(set1 & set2)