diff --git a/.env.example b/.env.example index 20c7aa1a..2b44f202 100644 --- a/.env.example +++ b/.env.example @@ -17,7 +17,7 @@ HOTKEY_NAME=default WANDB_API_KEY= # for issue bounties api calls GITTENSOR_VALIDATOR_PAT= -# Optional custom name for wandb logging +# Optional custom name for wandb logging WANDB_VALIDATOR_NAME=vali # ******* MINER VARIABLES ******* diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE/default.md similarity index 100% rename from .github/PULL_REQUEST_TEMPLATE.md rename to .github/PULL_REQUEST_TEMPLATE/default.md diff --git a/.github/PULL_REQUEST_TEMPLATE/weight_adjustment.md b/.github/PULL_REQUEST_TEMPLATE/weight_adjustment.md new file mode 100644 index 00000000..4dbb726f --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/weight_adjustment.md @@ -0,0 +1,52 @@ +## Weight Adjustment PR + +### Changes Summary + +| Metric | Gold | Silver | Bronze | Total | +| -------------------- | ---- | ------ | ------ | ----- | +| Repositories Added | 0 | 0 | 0 | 0 | +| Repositories Removed | 0 | 0 | 0 | 0 | +| Weights Modified | 0 | 0 | 0 | 0 | +| Net Weight Change | 0 | 0 | 0 | 0 | + +### Added Repositories + + + +| Repository | Tier | Branch | Weight | +| ---------- | ------ | ------ | ------ | +| owner/repo | silver | main | 20.00 | + +### Removed Repositories + + + +| Repository | Tier | Reason | +| ---------- | ------ | ------ | +| owner/repo | silver | — | + +### Justification + + + +### Additional Acceptable Branches + + + +- [ ] No additional_acceptable_branches changes in this PR +- [ ] Proof of merged PR(s) provided above for all additional_acceptable_branches entries + +### Checklist + +- [ ] Changes summary table is filled in accurately +- [ ] Net weight changes are justified in the Justification section +- [ ] Added repositories have correct tier, branch, and initial weight diff --git a/.gitignore b/.gitignore index 53a4d77f..7992e30f 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,12 @@ wandb *.log +# Merge predictions local DB +merge-prediction-data/ +gt-merge-preds.db +gt-merge-preds.db-wal +gt-merge-preds.db-shm + CLAUDE.md .claude/ .vscode/ diff --git a/docker-compose.vali.yml b/docker-compose.vali.yml index 0842abaf..a663f87d 100644 --- a/docker-compose.vali.yml +++ b/docker-compose.vali.yml @@ -12,6 +12,7 @@ services: volumes: # 'ro' = readonly - ${WALLET_PATH}:/root/.bittensor/wallets:ro + - ./merge-prediction-data:/app/data # optional: uncomment this if you are running validator database # networks: # - gittensor_network diff --git a/gittensor/__init__.py b/gittensor/__init__.py index ed045292..3b9ee569 100644 --- a/gittensor/__init__.py +++ b/gittensor/__init__.py @@ -16,6 +16,6 @@ # DEALINGS IN THE SOFTWARE. # NOTE: bump this value when updating the codebase -__version__ = '4.0.0' +__version__ = '5.0.0' version_split = __version__.split('.') __spec_version__ = (1000 * int(version_split[0])) + (10 * int(version_split[1])) + (1 * int(version_split[2])) diff --git a/gittensor/classes.py b/gittensor/classes.py index b27645ab..bd4e5520 100644 --- a/gittensor/classes.py +++ b/gittensor/classes.py @@ -10,7 +10,7 @@ from gittensor.constants import MIN_TOKEN_SCORE_FOR_BASE_SCORE from gittensor.utils.utils import parse_repo_name -from gittensor.validator.configurations.tier_config import Tier, TierConfig, TierStats +from gittensor.validator.oss_contributions.tier_config import Tier, TierConfig, TierStats GITHUB_DOMAIN = 'https://github.com/' diff --git a/gittensor/cli/issue_commands/helpers.py b/gittensor/cli/issue_commands/helpers.py index 1b5a2f63..18608aa5 100644 --- a/gittensor/cli/issue_commands/helpers.py +++ b/gittensor/cli/issue_commands/helpers.py @@ -21,6 +21,7 @@ import click from rich.console import Console from rich.panel import Panel +from substrateinterface import SubstrateInterface from gittensor.cli.issue_commands.tables import build_pr_table from gittensor.constants import CONTRACT_ADDRESS @@ -43,7 +44,6 @@ 'Cancelled': 'dim', } -ISSUE_BOUNTY_ELIGIBLE_STATUSES = {'registered', 'active'} # Default paths GITTENSOR_DIR = Path.home() / '.gittensor' @@ -193,16 +193,21 @@ def print_issue_submission_table( console.print(f'Showing {len(pull_requests)} submissions{suffix}') -def verify_miner_registration(ws_endpoint: str, contract_addr: str, hotkey_ss58: str) -> bool: - """Return whether the hotkey is registered on the subnet configured by the contract netuid.""" - import bittensor as bt - from substrateinterface import SubstrateInterface +def resolve_netuid_from_contract(ws_endpoint: str, contract_addr: str) -> Optional[int]: + """Read the subnet netuid stored in the on-chain contract.""" substrate = SubstrateInterface(url=ws_endpoint) packed = _read_contract_packed_storage(substrate, contract_addr) - netuid = None if packed and packed.get('netuid') is not None: - netuid = int(packed['netuid']) + return int(packed['netuid']) + return None + + +def verify_miner_registration(ws_endpoint: str, contract_addr: str, hotkey_ss58: str) -> bool: + """Return whether the hotkey is registered on the subnet configured by the contract netuid.""" + import bittensor as bt + + netuid = resolve_netuid_from_contract(ws_endpoint, contract_addr) if netuid is None: return False @@ -796,10 +801,9 @@ def fetch_issue_from_contract( ws_endpoint: str, contract_addr: str, issue_id: int, - require_active: bool = False, verbose: bool = False, ) -> Dict[str, Any]: - """Resolve an on-chain issue and validate bountied/active status.""" + """Resolve an on-chain issue and validate bountied status.""" issues = read_issues_from_contract(ws_endpoint, contract_addr, verbose) issue = next((i for i in issues if i.get('id') == issue_id), None) if not issue: @@ -807,10 +811,8 @@ def fetch_issue_from_contract( status = issue.get('status') or '' status_normalized = str(status).strip().lower() - if status_normalized not in ISSUE_BOUNTY_ELIGIBLE_STATUSES: + if status_normalized not in {'registered', 'active'}: raise click.ClickException(f'Issue #{issue_id} is not in a bountied state (status: {status}).') - if require_active and status_normalized != 'active': - raise click.ClickException(f'Issue #{issue_id} is not active (status: {status}).') repo = issue.get('repository_full_name', '') issue_number = issue.get('issue_number', 0) diff --git a/gittensor/cli/issue_commands/predict.py b/gittensor/cli/issue_commands/predict.py index f5b5fd75..0e9c8f83 100644 --- a/gittensor/cli/issue_commands/predict.py +++ b/gittensor/cli/issue_commands/predict.py @@ -7,6 +7,8 @@ import click +from gittensor.miner.broadcast import broadcast_predictions + from .help import StyledCommand from .helpers import ( _is_interactive, @@ -24,6 +26,7 @@ print_network_header, print_success, print_warning, + resolve_netuid_from_contract, resolve_network, success_panel, validate_issue_id, @@ -137,6 +140,10 @@ def issues_predict( ws_endpoint, network_name = resolve_network(network, rpc_url) effective_wallet, effective_hotkey = _resolve_wallet_identity(wallet_name, wallet_hotkey) + netuid = resolve_netuid_from_contract(ws_endpoint, contract_addr) + if netuid is None: + handle_exception(as_json, 'Could not resolve netuid from contract.') + if not as_json: print_network_header(network_name, contract_addr) console.print(f'Wallet: {effective_wallet}/{effective_hotkey}\n') @@ -173,17 +180,7 @@ def issues_predict( print_warning('Prediction cancelled') return - # 7) Interactive mode: verify miner first to avoid wasting manual input. - if is_interactive_mode: - _resolve_registered_miner_hotkey( - wallet_name=effective_wallet, - wallet_hotkey=effective_hotkey, - ws_endpoint=ws_endpoint, - contract_addr=contract_addr, - as_json=as_json, - ) - - # 8) Collect predictions by mode; validate PR membership for non-interactive modes. + # 7) Collect predictions by mode; validate PR membership for non-interactive modes. try: if is_interactive_mode: predictions = _collect_predictions_interactive(pull_requests) @@ -193,29 +190,15 @@ def issues_predict( except (click.ClickException, click.BadParameter) as e: handle_exception(as_json, str(e)) - # 9) Single/batch modes: verify miner after prediction payload validation. - if not is_interactive_mode: - _resolve_registered_miner_hotkey( - wallet_name=effective_wallet, - wallet_hotkey=effective_hotkey, - ws_endpoint=ws_endpoint, - contract_addr=contract_addr, - as_json=as_json, - ) - - payload = build_prediction_payload( - issue_id=issue_id, - repository=repo_full_name, - predictions=predictions, - ) - - # 10) Emit machine output or interactive confirmation flow. - if as_json: - emit_json(payload, pretty=True) - broadcast_predictions_stub(payload) - return + payload = { + 'issue_id': issue_id, + 'repository': repo_full_name, + 'predictions': dict(predictions), + 'github_access_token': '***', + } - if is_interactive_mode: + # 8) Confirmation prompt (interactive only). + if not as_json and is_interactive_mode: lines = format_prediction_lines(predictions) confirm_panel(lines, title='Prediction Confirmation') skip_confirm = yes or not _is_interactive() @@ -223,9 +206,35 @@ def issues_predict( print_warning('Prediction cancelled') return - success_panel(json_mod.dumps(payload, indent=2), title='Prediction Payload') - print_success('Prediction prepared (TODO: broadcast)') - broadcast_predictions_stub(payload) + # 9) Verify miner registration before broadcasting. + _resolve_registered_miner_hotkey( + wallet_name=effective_wallet, + wallet_hotkey=effective_hotkey, + ws_endpoint=ws_endpoint, + contract_addr=contract_addr, + as_json=as_json, + ) + + # 10) Show payload and broadcast to validators. + if as_json: + emit_json(payload, pretty=True) + + if not as_json: + success_panel(json_mod.dumps(payload, indent=2), title='Prediction Synapse') + + with loading_context('Broadcasting predictions to validators...', as_json): + results = broadcast_predictions( + payload=payload, + wallet_name=effective_wallet, + wallet_hotkey=effective_hotkey, + ws_endpoint=ws_endpoint, + netuid=netuid, + ) + + if as_json: + emit_json(results, pretty=True) + else: + _print_broadcast_results(results) def validate_probability(value: float, param_hint: str = 'probability') -> float: @@ -285,9 +294,7 @@ def _resolve_issue_context( """Load and validate on-chain issue context for prediction.""" try: with loading_context('Reading issues from contract...', as_json): - issue = fetch_issue_from_contract( - ws_endpoint, contract_addr, issue_id, require_active=True, verbose=verbose - ) + issue = fetch_issue_from_contract(ws_endpoint, contract_addr, issue_id, verbose=verbose) except click.ClickException as e: handle_exception(as_json, str(e)) @@ -361,22 +368,22 @@ def format_prediction_lines(predictions: dict[int, float]) -> str: return '\n'.join(lines) -def build_prediction_payload( - issue_id: int, - repository: str, - predictions: dict[int, float], -) -> dict[str, object]: - """Build validated payload for future network broadcast.""" - return { - 'issue_id': issue_id, - 'repository': repository, - 'predictions': dict(predictions), - } - +def _print_broadcast_results(results: dict[str, object]) -> None: + """Print broadcast results in human-readable format.""" + if results.get('error'): + print_error(str(results['error'])) + return + if results.get('success'): + print_success(f'Prediction accepted by {results["accepted"]}/{results["total_validators"]} validator(s)') + else: + print_error( + f'Prediction rejected or unreachable: {results["rejected"]}/{results["total_validators"]} validator(s)' + ) -def broadcast_predictions_stub(payload: dict[str, object]) -> None: - """Broadcast integration seam (stub).""" - pass + for r in results.get('results', []): + status = 'accepted' if r['accepted'] else 'rejected' + reason = f' ({r["rejection_reason"]})' if r.get('rejection_reason') else '' + console.print(f' {r["validator"]}... {status}{reason}') def _collect_predictions_interactive(prs: list[dict]) -> dict[int, float]: diff --git a/gittensor/cli/issue_commands/submissions.py b/gittensor/cli/issue_commands/submissions.py index d1eea984..2671f834 100644 --- a/gittensor/cli/issue_commands/submissions.py +++ b/gittensor/cli/issue_commands/submissions.py @@ -83,9 +83,7 @@ def issues_submissions( try: with loading_context('Fetching issue from contract...', as_json): - issue = fetch_issue_from_contract( - ws_endpoint, contract_addr, issue_id, require_active=False, verbose=verbose - ) + issue = fetch_issue_from_contract(ws_endpoint, contract_addr, issue_id, verbose=verbose) except click.ClickException as e: handle_exception(as_json, str(e)) diff --git a/gittensor/constants.py b/gittensor/constants.py index 58e9a521..bac5752f 100644 --- a/gittensor/constants.py +++ b/gittensor/constants.py @@ -139,4 +139,25 @@ # ============================================================================= CONTRACT_ADDRESS = '5FWNdk8YNtNcHKrAx2krqenFrFAZG7vmsd2XN2isJSew3MrD' ISSUES_TREASURY_UID = 111 # UID of the smart contract neuron, if set to RECYCLE_UID then it's disabled -ISSUES_TREASURY_EMISSION_SHARE = 0.15 # % of emissions routed to funding issues treasury +ISSUES_TREASURY_EMISSION_SHARE = 0.15 # % of emissions allocated to funding issues treasury + +# ============================================================================= +# Merge Predictions +# ============================================================================= +PREDICTIONS_EMISSIONS_SHARE = 0.15 # % of emissions allocated to prediction competition + +PREDICTIONS_EMA_BETA = 0.1 # EMA decay rate for predictions record +PREDICTIONS_CORRECTNESS_EXPONENT = 3 # exponent on correctness to harshly punish incorrect predictions +PREDICTIONS_TIMELINESS_EXPONENT = 1.8 # curve for early prediction bonus. higher = sharper curve. 1.0 = linear +PREDICTIONS_MAX_TIMELINESS_BONUS = 0.75 # max bonus for earliest predictions +PREDICTIONS_MAX_CONSENSUS_BONUS = 0.25 # max bonus for pre-convergence predictions +PREDICTIONS_MAX_ORDER_BONUS = 0.75 # max bonus for first correct predictor (applies to merged PR only) +PREDICTIONS_ORDER_CORRECTNESS_THRESHOLD = 0.66 # min raw correctness to qualify for order bonus +# variance threshold for full rewards +# if variance across predictions never exceeds this threshold, the solution must be 'obvious' +PREDICTIONS_CONSENSUS_VARIANCE_TARGET = 0.2 + +# Cooldown & Limits +PREDICTIONS_COOLDOWN_SECONDS = 900 # 15 min cooldown per miner per PR re-prediction +PREDICTIONS_MIN_VALUE = 0.0 +PREDICTIONS_MAX_VALUE = 1.0 diff --git a/gittensor/miner/broadcast.py b/gittensor/miner/broadcast.py new file mode 100644 index 00000000..e276daae --- /dev/null +++ b/gittensor/miner/broadcast.py @@ -0,0 +1,85 @@ +# Entrius 2025 + +"""Broadcast PredictionSynapse from miner to all validator axons.""" + +import asyncio + +import bittensor as bt + +from gittensor.miner.token_mgmt import load_token +from gittensor.synapses import PredictionSynapse + + +def broadcast_predictions( + payload: dict[str, object], + wallet_name: str, + wallet_hotkey: str, + ws_endpoint: str, + netuid: int, +) -> dict[str, object]: + """Broadcast PredictionSynapse to all validator axons via dendrite. + + Args: + payload: Dict with issue_id, repository, predictions. + wallet_name: Bittensor wallet name. + wallet_hotkey: Bittensor hotkey name. + ws_endpoint: Subtensor RPC endpoint. + netuid: Subnet UID to broadcast on. + + Returns: + Dict with success, total_validators, accepted, rejected, results. + """ + github_pat = load_token(quiet=True) + if not github_pat: + return {'success': False, 'error': 'GITTENSOR_MINER_PAT not set or invalid.', 'results': []} + + wallet = bt.Wallet(name=wallet_name, hotkey=wallet_hotkey) + subtensor = bt.Subtensor(network=ws_endpoint) + metagraph = subtensor.metagraph(netuid=netuid) + dendrite = bt.Dendrite(wallet=wallet) + + synapse = PredictionSynapse( + github_access_token=github_pat, + issue_id=int(payload['issue_id']), + repository=str(payload['repository']), + predictions={int(k): float(v) for k, v in payload['predictions'].items()}, + ) + + # Get axons for high-trust validators (> 0.6 vtrust) with permit that are actively serving. + validator_axons = [ + axon + for uid, axon in enumerate(metagraph.axons) + if metagraph.validator_permit[uid] and axon.is_serving and float(metagraph.Tv[uid]) > 0.6 + ] + + if not validator_axons: + return {'success': False, 'error': 'No reachable validator axons found on the network.', 'results': []} + + responses = asyncio.get_event_loop().run_until_complete( + dendrite( + axons=validator_axons, + synapse=synapse, + deserialize=False, + timeout=12.0, + ) + ) + + results = [] + for axon, resp in zip(validator_axons, responses): + results.append( + { + 'validator': axon.hotkey[:16], + 'accepted': resp.accepted if hasattr(resp, 'accepted') else None, + 'rejection_reason': resp.rejection_reason if hasattr(resp, 'rejection_reason') else None, + 'status_code': resp.dendrite.status_code if hasattr(resp, 'dendrite') else None, + } + ) + + accepted_count = sum(1 for r in results if r['accepted'] is True) + return { + 'success': accepted_count > 0, + 'total_validators': len(validator_axons), + 'accepted': accepted_count, + 'rejected': len(results) - accepted_count, + 'results': results, + } diff --git a/gittensor/miner/token_mgmt.py b/gittensor/miner/token_mgmt.py index 90da4f36..29ba0780 100644 --- a/gittensor/miner/token_mgmt.py +++ b/gittensor/miner/token_mgmt.py @@ -29,27 +29,31 @@ def init() -> bool: return True -def load_token() -> Optional[str]: +def load_token(quiet: bool = False) -> Optional[str]: """ Load GitHub token from environment variable Returns: Optional[str]: The GitHub access token string if valid, None otherwise """ - bt.logging.info('Loading GitHub token from environment.') + if not quiet: + bt.logging.info('Loading GitHub token from environment.') access_token = os.getenv('GITTENSOR_MINER_PAT') if not access_token: - bt.logging.error('No GitHub token found in GITTENSOR_MINER_PAT environment variable!') + if not quiet: + bt.logging.error('No GitHub token found in GITTENSOR_MINER_PAT environment variable!') return None # Test if token is still valid if is_token_valid(access_token): - bt.logging.info('GitHub token loaded successfully and is valid.') + if not quiet: + bt.logging.info('GitHub token loaded successfully and is valid.') return access_token - bt.logging.error('GitHub token is invalid or expired.') + if not quiet: + bt.logging.error('GitHub token is invalid or expired.') return None diff --git a/gittensor/synapses.py b/gittensor/synapses.py index b54f5762..f46249f3 100644 --- a/gittensor/synapses.py +++ b/gittensor/synapses.py @@ -14,3 +14,31 @@ class GitPatSynapse(bt.Synapse): """ github_access_token: Optional[str] = None + + +class PredictionSynapse(bt.Synapse): + """Miner-initiated push synapse for merge predictions. + + Request fields (set by miner): + - github_access_token: Miner's GitHub PAT for identity verification and account age check. + - issue_id: On-chain issue ID (NOT GitHub issue number). + - repository: Full repo name, e.g. "entrius/gittensor". + - predictions: Mapping of PR number -> probability (0.0-1.0). + Sum across all of a miner's predictions for an issue must be <= 1.0. + Each submission can contain one or many PR predictions. + Submitting a prediction for a PR that already has one overwrites it. + + Response fields (set by validator): + - accepted: Whether the prediction was stored. + - rejection_reason: Human-readable reason if rejected. + """ + + # Miner Request + github_access_token: str + issue_id: int + repository: str + predictions: dict[int, float] + + # Validator Response + accepted: Optional[bool] = None + rejection_reason: Optional[str] = None diff --git a/gittensor/utils/config.py b/gittensor/utils/config.py index f81e7fa0..2de04c0e 100644 --- a/gittensor/utils/config.py +++ b/gittensor/utils/config.py @@ -230,13 +230,6 @@ def add_validator_args(cls, parser): default='opentensor-dev', ) - parser.add_argument( - '--neuron.remote_debug_port', - type=int, - help='FOR DEVELOPMENT: Port for remote debugging API endpoint. If set, enables debug API on this port and debugpy on port+1.', - default=None, - ) - def config(cls): """ diff --git a/gittensor/utils/github_api_tools.py b/gittensor/utils/github_api_tools.py index fe676d07..f1e9c224 100644 --- a/gittensor/utils/github_api_tools.py +++ b/gittensor/utils/github_api_tools.py @@ -868,6 +868,41 @@ def load_miners_prs( ) +def get_pr_open_times(repo: str, pr_numbers: List[int], token: str) -> Dict[int, datetime]: + """Fetch PR creation dates from GitHub API. + + Args: + repo: Repository full name (e.g., 'owner/repo') + pr_numbers: List of PR numbers to fetch + token: GitHub PAT for authentication + + Returns: + Dict mapping pr_number to created_at datetime (UTC). PRs that fail + to fetch are omitted from the result. + """ + headers = make_headers(token) + result: Dict[int, datetime] = {} + + for pr_number in pr_numbers: + try: + response = requests.get( + f'{BASE_GITHUB_API_URL}/repos/{repo}/pulls/{pr_number}', + headers=headers, + timeout=15, + ) + if response.status_code == 200: + data = response.json() + created_at = data.get('created_at') + if created_at: + result[pr_number] = datetime.fromisoformat(created_at.rstrip('Z')).replace(tzinfo=timezone.utc) + else: + bt.logging.debug(f'Failed to fetch PR #{pr_number} from {repo}: status {response.status_code}') + except requests.exceptions.RequestException as e: + bt.logging.debug(f'Error fetching PR #{pr_number} from {repo}: {e}') + + return result + + def extract_pr_number_from_url(pr_url: str) -> Optional[int]: """Extract PR number from a GitHub PR URL. diff --git a/gittensor/validator/configurations/tier_config.py b/gittensor/validator/configurations/tier_config.py deleted file mode 100644 index 091be406..00000000 --- a/gittensor/validator/configurations/tier_config.py +++ /dev/null @@ -1,120 +0,0 @@ -from dataclasses import dataclass -from enum import Enum -from typing import Optional - -from gittensor.constants import ( - DEFAULT_COLLATERAL_PERCENT, - DEFAULT_MAX_CONTRIBUTION_SCORE_FOR_FULL_BONUS, - DEFAULT_MERGED_PR_BASE_SCORE, - MAX_CONTRIBUTION_BONUS, -) - - -@dataclass -class TierStats: - """Statistics for a single tier.""" - - merged_count: int = 0 - closed_count: int = 0 - open_count: int = 0 - - unique_repo_contribution_count: int = 0 - # Unique repos that meet a min token score threshold - qualified_unique_repo_count: int = 0 - - # Included as scoring details at the tier level - earned_score: float = 0.0 - collateral_score: float = 0.0 - - # Token scoring breakdown for this tier - token_score: float = 0.0 - structural_count: int = 0 - structural_score: float = 0.0 - leaf_count: int = 0 - leaf_score: float = 0.0 - - @property - def total_attempts(self) -> int: - return self.merged_count + self.closed_count - - @property - def total_prs(self) -> int: - return self.merged_count + self.closed_count + self.open_count - - @property - def credibility(self) -> float: - return self.merged_count / self.total_attempts if self.total_attempts > 0 else 0.0 - - -class Tier(str, Enum): - BRONZE = 'Bronze' - SILVER = 'Silver' - GOLD = 'Gold' - - -TIER_DEFAULTS = { - 'merged_pr_base_score': DEFAULT_MERGED_PR_BASE_SCORE, - 'contribution_score_for_full_bonus': DEFAULT_MAX_CONTRIBUTION_SCORE_FOR_FULL_BONUS, - 'contribution_score_max_bonus': MAX_CONTRIBUTION_BONUS, - 'open_pr_collateral_percentage': DEFAULT_COLLATERAL_PERCENT, -} - - -@dataclass(frozen=True) -class TierConfig: - required_credibility: Optional[float] - required_min_token_score: Optional[float] # Minimum total token score to unlock tier - # Unique repos with min token score requirement (both must be set or both None) - required_unique_repos_count: Optional[int] # Number of unique repos needed - required_min_token_score_per_repo: Optional[float] # Min token score each repo must have - - # Tier-specific scaling - credibility_scalar: int - - # Defaults (can override per-tier if needed) - merged_pr_base_score: int = TIER_DEFAULTS['merged_pr_base_score'] - contribution_score_for_full_bonus: int = TIER_DEFAULTS['contribution_score_for_full_bonus'] - contribution_score_max_bonus: int = TIER_DEFAULTS['contribution_score_max_bonus'] - open_pr_collateral_percentage: int = TIER_DEFAULTS['open_pr_collateral_percentage'] - - -TIERS: dict[Tier, TierConfig] = { - Tier.BRONZE: TierConfig( - required_credibility=0.70, - required_min_token_score=None, - required_unique_repos_count=3, - required_min_token_score_per_repo=5.0, # At least n initial unique repos must have at least x token score - credibility_scalar=1.0, - ), - Tier.SILVER: TierConfig( - required_credibility=0.65, - required_min_token_score=300.0, # Minimum total token score for Silver unlock - required_unique_repos_count=3, - required_min_token_score_per_repo=89.0, # At least n repos must have at least x token score - credibility_scalar=1.5, - ), - Tier.GOLD: TierConfig( - required_credibility=0.60, - required_min_token_score=500.0, # Minimum total token score for Gold unlock - required_unique_repos_count=3, - required_min_token_score_per_repo=144.0, # At least n unique repos must have at least x token score - credibility_scalar=2.0, - ), -} -TIERS_ORDER: list[Tier] = list(TIERS.keys()) - - -def get_next_tier(current: Tier) -> Optional[Tier]: - """Returns the next tier, or None if already at top.""" - idx = TIERS_ORDER.index(current) - if idx + 1 < len(TIERS_ORDER): - return TIERS_ORDER[idx + 1] - return None - - -def get_tier_from_config(tier_config: TierConfig) -> Optional[Tier]: - """Reverse lookup tier from TierConfig.""" - for tier, config in TIERS.items(): - if config == tier_config: - return tier - return None diff --git a/gittensor/validator/forward.py b/gittensor/validator/forward.py index e19cce8f..c8864806 100644 --- a/gittensor/validator/forward.py +++ b/gittensor/validator/forward.py @@ -2,262 +2,147 @@ # Copyright © 2025 Entrius import asyncio -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING, Dict, Tuple import bittensor as bt +import numpy as np from gittensor.classes import MinerEvaluation -from gittensor.constants import ISSUES_TREASURY_EMISSION_SHARE, ISSUES_TREASURY_UID +from gittensor.constants import ISSUES_TREASURY_EMISSION_SHARE, ISSUES_TREASURY_UID, PREDICTIONS_EMISSIONS_SHARE +from gittensor.utils.uids import get_all_uids +from gittensor.validator.issue_competitions.forward import issue_competitions +from gittensor.validator.merge_predictions.settlement import merge_predictions +from gittensor.validator.oss_contributions.reward import get_rewards +from gittensor.validator.utils.config import VALIDATOR_STEPS_INTERVAL, VALIDATOR_WAIT from gittensor.validator.utils.load_weights import ( load_master_repo_weights, load_programming_language_weights, load_token_config, ) -# ADD THIS for proper type hinting to navigate code easier. if TYPE_CHECKING: - from neurons.base.validator import BaseValidatorNeuron - -# Issue bounties integration -from gittensor.utils.github_api_tools import check_github_issue_closed -from gittensor.utils.uids import get_all_uids -from gittensor.validator.evaluation.reward import get_rewards -from gittensor.validator.issue_competitions.contract_client import IssueCompetitionContractClient, IssueStatus -from gittensor.validator.utils.config import GITTENSOR_VALIDATOR_PAT, VALIDATOR_STEPS_INTERVAL, VALIDATOR_WAIT -from gittensor.validator.utils.issue_competitions import ( - get_contract_address, - get_miner_coldkey, -) + from neurons.validator import Validator -async def forward(self: 'BaseValidatorNeuron') -> None: +async def forward(self: 'Validator') -> None: """Execute the validator's forward pass. Performs the core validation cycle every VALIDATOR_STEPS_INTERVAL steps: - 1. Get all available miner UIDs - 2. Score OSS contributions and get miner evaluations - 3. Update scores using exponential moving average - 4. Run issue bounties verification (needs tier data from scoring) - - Args: - self: The validator instance containing all necessary state + 1. Score OSS contributions (pure scoring, no side effects) + 2. Run issue bounties verification (needs tier data from scoring) + 3. Settle merge predictions (score + update EMAs) + 4. Build blended rewards array across all emission sources + 5. Update scores with blended rewards + + Emission blending: + - OSS contributions: 70% (1.0 - treasury - predictions) + - Issue bounties treasury: 15% flat to treasury UID + - Merge predictions: 15% distributed by EMA scores """ if self.step % VALIDATOR_STEPS_INTERVAL == 0: miner_uids = get_all_uids(self) - # Score OSS contributions - returns evaluations for issue verification - miner_evaluations = await oss_contributions(self, miner_uids) + rewards, miner_evaluations = await oss_contributions(self, miner_uids) - # Issue bounties verification - await issues_competition(self, miner_evaluations) + await issue_competitions(self, miner_evaluations) - await asyncio.sleep(VALIDATOR_WAIT) + await merge_predictions(self, miner_evaluations) + # Build blended rewards array across all emission sources + oss_share = 1.0 - ISSUES_TREASURY_EMISSION_SHARE - PREDICTIONS_EMISSIONS_SHARE + rewards *= oss_share -async def oss_contributions(self: 'BaseValidatorNeuron', miner_uids: set[int]) -> Dict[int, MinerEvaluation]: - """Score OSS contributions and return miner evaluations for downstream use.""" - master_repositories = load_master_repo_weights() - programming_languages = load_programming_language_weights() - token_config = load_token_config() + if ISSUES_TREASURY_UID > 0 and ISSUES_TREASURY_UID in miner_uids: + sorted_uids = sorted(miner_uids) + treasury_idx = sorted_uids.index(ISSUES_TREASURY_UID) + rewards[treasury_idx] = ISSUES_TREASURY_EMISSION_SHARE - tree_sitter_count = sum(1 for c in token_config.language_configs.values() if c.language is not None) + bt.logging.info( + f'Treasury allocation: Smart Contract UID {ISSUES_TREASURY_UID} receives ' + f'{ISSUES_TREASURY_EMISSION_SHARE * 100:.0f}% of emissions' + ) - bt.logging.info('***** Starting scoring round *****') - bt.logging.info(f'Total Repositories loaded: {len(master_repositories)}') - bt.logging.info(f'Total Languages loaded: {len(programming_languages)}') - bt.logging.info(f'Token config: {tree_sitter_count} tree-sitter languages') - bt.logging.info(f'Neurons to evaluate: {len(miner_uids)}') + prediction_rewards = build_prediction_ema_rewards(self, miner_uids, miner_evaluations) + rewards += prediction_rewards - rewards, miner_evaluations = await get_rewards( - self, miner_uids, master_repositories, programming_languages, token_config - ) + bt.logging.info( + f'Blended rewards: OSS {oss_share * 100:.0f}% + treasury {ISSUES_TREASURY_EMISSION_SHARE * 100:.0f}% ' + f'+ predictions {PREDICTIONS_EMISSIONS_SHARE * 100:.0f}% ' + f'(prediction sum={prediction_rewards.sum():.4f})' + ) - # ------------------------------------------------------------------------- - # Issue Bounties Treasury Allocation - # The smart contract neuron (ISSUES_TREASURY_UID) accumulates emissions - # which fund issue bounty payouts. We allocate a fixed percentage of - # total emissions to this treasury by scaling down all miner rewards - # and assigning the remainder to the treasury UID. - # ------------------------------------------------------------------------- - if ISSUES_TREASURY_UID > 0 and ISSUES_TREASURY_UID in miner_uids: - treasury_share = ISSUES_TREASURY_EMISSION_SHARE - miner_share = 1.0 - treasury_share + self.update_scores(rewards, miner_uids) - # rewards array is indexed by position in sorted(miner_uids) - sorted_uids = sorted(miner_uids) - treasury_idx = sorted_uids.index(ISSUES_TREASURY_UID) + await asyncio.sleep(VALIDATOR_WAIT) - # Scale down all rewards proportionally - rewards *= miner_share - # Assign treasury's share - rewards[treasury_idx] = treasury_share +def build_prediction_ema_rewards( + self: 'Validator', + miner_uids: set[int], + miner_evaluations: Dict[int, MinerEvaluation], +) -> np.ndarray: + """Build rewards array from prediction EMA scores, scaled to PREDICTIONS_EMISSIONS_SHARE. - bt.logging.info( - f'Treasury allocation: Smart Contract UID {ISSUES_TREASURY_UID} receives ' - f'{treasury_share * 100:.0f}% of emissions, miners share {miner_share * 100:.0f}%' - ) + Maps github_id-keyed EMAs back to UIDs via miner_evaluations. + """ + sorted_uids = sorted(miner_uids) + prediction_rewards = np.zeros(len(sorted_uids), dtype=np.float64) - self.update_scores(rewards, miner_uids) + all_emas = self.mp_storage.get_all_emas() + if not all_emas: + return prediction_rewards - return miner_evaluations + # Build github_id -> uid mapping from current miner evaluations + # NOTE: detect_and_penalize_miners_sharing_github() already zeroes github_id + # for duplicate accounts before this runs, so the '!= 0' filter handles them. + github_id_to_uid: Dict[str, int] = {} + for uid, evaluation in miner_evaluations.items(): + if evaluation and evaluation.github_id and evaluation.github_id != '0': + github_id_to_uid[evaluation.github_id] = uid + for mp_record in all_emas: + github_id = mp_record['github_id'] + ema_score = mp_record['ema_score'] -async def issues_competition( - self: 'BaseValidatorNeuron', - miner_evaluations: Dict[int, MinerEvaluation], -) -> None: - """ - Run the issue bounties forward pass. + if ema_score <= 0: + continue - 1. Harvest emissions into the bounty pool - 2. Get active issues from the smart contract - 3. For each active issue, check GitHub: - - If solved by bronze+ miner -> vote_solution - - If closed but not by eligible miner -> vote_cancel_issue + uid = github_id_to_uid.get(github_id) + if uid is None or uid not in miner_uids: + continue - Args: - self: The validator instance - miner_evaluations: Fresh scoring data from oss_contributions(), keyed by UID - """ - try: - if not GITTENSOR_VALIDATOR_PAT: - bt.logging.info('GITTENSOR_VALIDATOR_PAT not set, skipping issue bounties voting entirely.') - return - - contract_addr = get_contract_address() - if not contract_addr: - bt.logging.warning('Issue bounties: no contract address configured') - return - - bt.logging.info('***** Starting Issue Bounties *****') - bt.logging.info(f'Contract address: {contract_addr}') - - # Create contract client - contract_client = IssueCompetitionContractClient( - contract_address=contract_addr, - subtensor=self.subtensor, - ) + idx = sorted_uids.index(uid) + prediction_rewards[idx] = ema_score - # Harvest emissions first - flush accumulated stake into bounty pool - harvest_result = contract_client.harvest_emissions(self.wallet) - if harvest_result and harvest_result.get('status') == 'success': - bt.logging.success(f'Harvested emissions! Extrinsic: {harvest_result.get("tx_hash", "")}') - - # Build mapping of github_id->hotkey for bronze+ miners only (eligible for payouts) - eligible_miners = { - eval.github_id: eval.hotkey - for eval in miner_evaluations.values() - if eval.github_id and eval.github_id != '0' and eval.current_tier is not None - } - bt.logging.info( - f'Issue bounties: {len(eligible_miners)} eligible miners (bronze+) out of {len(miner_evaluations)} total' - ) - for github_id, hotkey in eligible_miners.items(): - bt.logging.info(f' Eligible miner: github_id={github_id}, hotkey={hotkey[:12]}...') + # Normalize to sum=1.0, then scale to prediction share + total = prediction_rewards.sum() + if total > 0: + prediction_rewards = (prediction_rewards / total) * PREDICTIONS_EMISSIONS_SHARE - # Get active issues from contract - active_issues = contract_client.get_issues_by_status(IssueStatus.ACTIVE) - bt.logging.info(f'Found {len(active_issues)} active issues') + return prediction_rewards - votes_cast = 0 - cancels_cast = 0 - errors = [] - for issue in active_issues: - bounty_display = issue.bounty_amount / 1e9 - issue_label = ( - f'{issue.repository_full_name}#{issue.issue_number} (id={issue.id}, bounty={bounty_display:.2f} ALPHA)' - ) - try: - bt.logging.info(f'--- Processing issue: {issue_label} ---') - - github_state = check_github_issue_closed( - issue.repository_full_name, issue.issue_number, GITTENSOR_VALIDATOR_PAT - ) - - if github_state is None: - bt.logging.warning(f'Could not check GitHub state for {issue_label}') - continue - - if not github_state.get('is_closed'): - bt.logging.info(f'Issue still open on GitHub: {issue_label}') - continue - - solver_github_id = github_state.get('solver_github_id') - pr_number = github_state.get('pr_number') - bt.logging.info( - f'Issue closed on GitHub: {issue_label} | solver_github_id={solver_github_id}, pr_number={pr_number}' - ) - - if not solver_github_id: - bt.logging.info(f'No identifiable solver, voting cancel: {issue_label}') - success = contract_client.vote_cancel_issue( - issue_id=issue.id, - reason='Issue closed without identifiable solver', - wallet=self.wallet, - ) - if success: - cancels_cast += 1 - bt.logging.info(f'Voted cancel (no solver): {issue_label}') - continue - - miner_hotkey = eligible_miners.get(str(solver_github_id)) - if not miner_hotkey: - bt.logging.info(f'Solver {solver_github_id} not in eligible miners, voting cancel: {issue_label}') - success = contract_client.vote_cancel_issue( - issue_id=issue.id, - reason=f'Issue closed externally (not by eligible miner, solver: {solver_github_id})', - wallet=self.wallet, - ) - if success: - cancels_cast += 1 - bt.logging.info(f'Voted cancel (solver {solver_github_id} not eligible): {issue_label}') - continue - - miner_coldkey = get_miner_coldkey(miner_hotkey, self.subtensor, self.config.netuid) - if not miner_coldkey: - bt.logging.warning( - f'Could not get coldkey for hotkey {miner_hotkey} (solver {solver_github_id}): {issue_label}' - ) - continue - - bt.logging.info( - f'Voting solution: {issue_label} | PR#{pr_number}, solver={solver_github_id}, hotkey={miner_hotkey[:12]}...' - ) - success = contract_client.vote_solution( - issue_id=issue.id, - solver_hotkey=miner_hotkey, - solver_coldkey=miner_coldkey, - pr_number=pr_number or 0, - wallet=self.wallet, - ) - if success: - votes_cast += 1 - bt.logging.success( - f'Voted solution for {issue_label}: hotkey={miner_hotkey[:12]}..., PR#{pr_number}' - ) - else: - bt.logging.warning(f'Vote solution call failed: {issue_label}') - errors.append(f'Vote failed for {issue_label}') - - except Exception as e: - bt.logging.error(f'Error processing {issue_label}: {e}') - errors.append(f'{issue_label}: {str(e)}') - - if errors: - bt.logging.warning(f'Issue bounties errors: {errors[:3]}') - - if votes_cast > 0 or cancels_cast > 0: - bt.logging.success( - f'=== Issue Bounties Complete: processed {len(active_issues)} issues, ' - f'{votes_cast} solution votes, {cancels_cast} cancel votes ===' - ) - else: - bt.logging.info( - f'***** Issue Bounties Complete: processed {len(active_issues)} issues (no state changes) *****' - ) +async def oss_contributions(self: 'Validator', miner_uids: set[int]) -> Tuple[np.ndarray, Dict[int, MinerEvaluation]]: + """Score OSS contributions and return raw rewards + miner evaluations. + + Pure scoring — no treasury allocation or weight updates. Those are + handled by the caller (forward()). + """ + master_repositories = load_master_repo_weights() + programming_languages = load_programming_language_weights() + token_config = load_token_config() + + tree_sitter_count = sum(1 for c in token_config.language_configs.values() if c.language is not None) + + bt.logging.info('***** Starting scoring round *****') + bt.logging.info(f'Total Repositories loaded: {len(master_repositories)}') + bt.logging.info(f'Total Languages loaded: {len(programming_languages)}') + bt.logging.info(f'Token config: {tree_sitter_count} tree-sitter languages') + bt.logging.info(f'Neurons to evaluate: {len(miner_uids)}') + + rewards, miner_evaluations = await get_rewards( + self, miner_uids, master_repositories, programming_languages, token_config + ) - except Exception as e: - bt.logging.error(f'Issue bounties forward failed: {e}') + return rewards, miner_evaluations diff --git a/gittensor/validator/issue_competitions/forward.py b/gittensor/validator/issue_competitions/forward.py new file mode 100644 index 00000000..bd8e1656 --- /dev/null +++ b/gittensor/validator/issue_competitions/forward.py @@ -0,0 +1,181 @@ +# The MIT License (MIT) +# Copyright © 2025 Entrius + +"""Issue bounties forward pass — harvest, verify, and vote on active issues.""" + +from typing import TYPE_CHECKING, Dict + +import bittensor as bt + +from gittensor.classes import MinerEvaluation +from gittensor.utils.github_api_tools import check_github_issue_closed +from gittensor.validator.issue_competitions.contract_client import IssueCompetitionContractClient, IssueStatus +from gittensor.validator.utils.config import GITTENSOR_VALIDATOR_PAT +from gittensor.validator.utils.issue_competitions import ( + get_contract_address, + get_miner_coldkey, +) + +if TYPE_CHECKING: + from neurons.base.validator import BaseValidatorNeuron + + +async def issue_competitions( + self: 'BaseValidatorNeuron', + miner_evaluations: Dict[int, MinerEvaluation], +) -> None: + """ + Run the issue bounties forward pass. + + 1. Harvest emissions into the bounty pool + 2. Get active issues from the smart contract + 3. For each active issue, check GitHub: + - If solved by bronze+ miner -> vote_solution + - If closed but not by eligible miner -> vote_cancel_issue + + Args: + self: The validator instance + miner_evaluations: Fresh scoring data from oss_contributions(), keyed by UID + """ + try: + if not GITTENSOR_VALIDATOR_PAT: + bt.logging.warning( + 'GITTENSOR_VALIDATOR_PAT not set, skipping issue bounties voting entirely. (This does NOT affect vtrust/consensus)' + ) + return + + contract_addr = get_contract_address() + if not contract_addr: + bt.logging.warning('Issue bounties: no contract address configured') + return + + bt.logging.info('***** Starting Issue Bounties *****') + bt.logging.info(f'Contract address: {contract_addr}') + + # Create contract client + contract_client = IssueCompetitionContractClient( + contract_address=contract_addr, + subtensor=self.subtensor, + ) + + # Harvest emissions first - flush accumulated stake into bounty pool + harvest_result = contract_client.harvest_emissions(self.wallet) + if harvest_result and harvest_result.get('status') == 'success': + bt.logging.success(f'Harvested emissions! Extrinsic: {harvest_result.get("tx_hash", "")}') + + # Build mapping of github_id->hotkey for bronze+ miners only (eligible for payouts) + eligible_miners = { + eval.github_id: eval.hotkey + for eval in miner_evaluations.values() + if eval.github_id and eval.github_id != '0' and eval.current_tier is not None + } + bt.logging.info( + f'Issue bounties: {len(eligible_miners)} eligible miners (bronze+) out of {len(miner_evaluations)} total' + ) + for github_id, hotkey in eligible_miners.items(): + bt.logging.info(f' Eligible miner: github_id={github_id}, hotkey={hotkey[:12]}...') + + # Get active issues from contract + active_issues = contract_client.get_issues_by_status(IssueStatus.ACTIVE) + bt.logging.info(f'Found {len(active_issues)} active issues') + + votes_cast = 0 + cancels_cast = 0 + errors = [] + + for issue in active_issues: + bounty_display = issue.bounty_amount / 1e9 + issue_label = ( + f'{issue.repository_full_name}#{issue.issue_number} (id={issue.id}, bounty={bounty_display:.2f} ALPHA)' + ) + try: + bt.logging.info(f'--- Processing issue: {issue_label} ---') + + github_state = check_github_issue_closed( + issue.repository_full_name, issue.issue_number, GITTENSOR_VALIDATOR_PAT + ) + + if github_state is None: + bt.logging.warning(f'Could not check GitHub state for {issue_label}') + continue + + if not github_state.get('is_closed'): + bt.logging.info(f'Issue still open on GitHub: {issue_label}') + continue + + solver_github_id = github_state.get('solver_github_id') + pr_number = github_state.get('pr_number') + bt.logging.info( + f'Issue closed on GitHub: {issue_label} | solver_github_id={solver_github_id}, pr_number={pr_number}' + ) + + if not solver_github_id: + bt.logging.info(f'No identifiable solver, voting cancel: {issue_label}') + success = contract_client.vote_cancel_issue( + issue_id=issue.id, + reason='Issue closed without identifiable solver', + wallet=self.wallet, + ) + if success: + cancels_cast += 1 + bt.logging.info(f'Voted cancel (no solver): {issue_label}') + continue + + miner_hotkey = eligible_miners.get(str(solver_github_id)) + if not miner_hotkey: + bt.logging.info(f'Solver {solver_github_id} not in eligible miners, voting cancel: {issue_label}') + success = contract_client.vote_cancel_issue( + issue_id=issue.id, + reason=f'Issue closed externally (not by eligible miner, solver: {solver_github_id})', + wallet=self.wallet, + ) + if success: + cancels_cast += 1 + bt.logging.info(f'Voted cancel (solver {solver_github_id} not eligible): {issue_label}') + continue + + miner_coldkey = get_miner_coldkey(miner_hotkey, self.subtensor, self.config.netuid) + if not miner_coldkey: + bt.logging.warning( + f'Could not get coldkey for hotkey {miner_hotkey} (solver {solver_github_id}): {issue_label}' + ) + continue + + bt.logging.info( + f'Voting solution: {issue_label} | PR#{pr_number}, solver={solver_github_id}, hotkey={miner_hotkey[:12]}...' + ) + success = contract_client.vote_solution( + issue_id=issue.id, + solver_hotkey=miner_hotkey, + solver_coldkey=miner_coldkey, + pr_number=pr_number or 0, + wallet=self.wallet, + ) + if success: + votes_cast += 1 + bt.logging.success( + f'Voted solution for {issue_label}: hotkey={miner_hotkey[:12]}..., PR#{pr_number}' + ) + else: + bt.logging.warning(f'Vote solution call failed: {issue_label}') + errors.append(f'Vote failed for {issue_label}') + + except Exception as e: + bt.logging.error(f'Error processing {issue_label}: {e}') + errors.append(f'{issue_label}: {str(e)}') + + if errors: + bt.logging.warning(f'Issue bounties errors: {errors[:3]}') + + if votes_cast > 0 or cancels_cast > 0: + bt.logging.success( + f'=== Issue Bounties Complete: processed {len(active_issues)} issues, ' + f'{votes_cast} solution votes, {cancels_cast} cancel votes ===' + ) + else: + bt.logging.info( + '***** Issue Bounties Complete: processed {len(active_issues)} issues (no state changes) *****' + ) + + except Exception as e: + bt.logging.error(f'Issue bounties forward failed: {e}') diff --git a/gittensor/validator/merge_predictions/__init__.py b/gittensor/validator/merge_predictions/__init__.py new file mode 100644 index 00000000..11d1b6b6 --- /dev/null +++ b/gittensor/validator/merge_predictions/__init__.py @@ -0,0 +1 @@ +# Entrius 2025 diff --git a/gittensor/validator/merge_predictions/checks.py b/gittensor/validator/merge_predictions/checks.py new file mode 100644 index 00000000..3ae283fa --- /dev/null +++ b/gittensor/validator/merge_predictions/checks.py @@ -0,0 +1,69 @@ +# Entrius 2025 + +"""External state checks for predictions (on-chain + GitHub).""" + +from typing import TYPE_CHECKING + +import bittensor as bt + +from gittensor.validator.issue_competitions.contract_client import ( + ContractIssue, + IssueCompetitionContractClient, + IssueStatus, +) +from gittensor.validator.utils.config import GITTENSOR_VALIDATOR_PAT + +if TYPE_CHECKING: + from neurons.validator import Validator + + +def check_issue_active(validator: 'Validator', issue_id: int) -> tuple[str | None, ContractIssue | None]: + """Verify issue is in a predictable state on-chain. Returns (error, issue).""" + try: + from gittensor.validator.utils.issue_competitions import get_contract_address + + contract_addr = get_contract_address() + if not contract_addr: + return 'Issue bounties not configured on this validator', None + + client = IssueCompetitionContractClient( + contract_address=contract_addr, + subtensor=validator.subtensor, + ) + issue = client.get_issue(issue_id) + if issue is None: + return f'Issue {issue_id} not found on-chain', None + if issue.status not in (IssueStatus.REGISTERED, IssueStatus.ACTIVE): + return f'Issue {issue_id} is not in a predictable state (status: {issue.status.name})', None + except Exception as e: + bt.logging.warning(f'Failed to check issue state for {issue_id}: {e}') + return f'Could not verify issue state: {e}', None + + return None, issue + + +def check_prs_open(repository: str, issue_number: int, predictions: dict[int, float]) -> tuple[str | None, set[int]]: + """Verify all predicted PRs are still open on GitHub. + + Returns (error, open_pr_numbers). open_pr_numbers is the full set of open PRs + for this issue, used downstream to exclude closed-PR predictions from probability totals. + """ + if not GITTENSOR_VALIDATOR_PAT: + bt.logging.warning('No GITTENSOR_VALIDATOR_PAT, skipping PR open check') + return None, set() + + try: + from gittensor.utils.github_api_tools import find_prs_for_issue + + open_prs = find_prs_for_issue(repository, issue_number, open_only=True, token=GITTENSOR_VALIDATOR_PAT) + open_pr_numbers = {pr.get('number') if isinstance(pr, dict) else getattr(pr, 'number', None) for pr in open_prs} + + for pr_number in predictions: + if pr_number not in open_pr_numbers: + return f'PR #{pr_number} is not open on {repository}', open_pr_numbers + + except Exception as e: + bt.logging.warning(f'Failed to check PR state for {repository}: {e}') + return None, set() + + return None, open_pr_numbers diff --git a/gittensor/validator/merge_predictions/handler.py b/gittensor/validator/merge_predictions/handler.py new file mode 100644 index 00000000..8f2b0d6d --- /dev/null +++ b/gittensor/validator/merge_predictions/handler.py @@ -0,0 +1,143 @@ +# Entrius 2025 + +"""Axon handler for PredictionSynapse. + +Attached to the validator's axon via functools.partial in Validator.__init__(). +Runs in the axon's FastAPI thread pool — fully parallel to the main scoring loop. +""" + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Tuple + +import bittensor as bt + +from gittensor.synapses import PredictionSynapse +from gittensor.validator.merge_predictions.checks import check_issue_active, check_prs_open +from gittensor.validator.merge_predictions.validation import validate_prediction_values +from gittensor.validator.utils.github_validation import validate_github_credentials + +if TYPE_CHECKING: + from neurons.validator import Validator + + +async def handle_prediction(validator: 'Validator', synapse: PredictionSynapse) -> PredictionSynapse: + """Validate and store a miner's prediction. Runs in axon thread pool.""" + + mp_storage = validator.mp_storage + miner_hotkey = synapse.dendrite.hotkey + uid = validator.metagraph.hotkeys.index(miner_hotkey) + + def _reject(reason: str) -> PredictionSynapse: + synapse.accepted = False + synapse.rejection_reason = reason + bt.logging.warning( + f'Merge prediction rejected — UID: {uid}, ID: {synapse.issue_id}, ' + f'repo: {synapse.repository}, PRs: {list(synapse.predictions.keys())}, ' + f'reason: {reason}' + ) + return synapse + + # 1) Verify issue is in a predictable state on-chain + error, issue = check_issue_active(validator, synapse.issue_id) + if error: + return _reject(error) + + # 2) Verify predicted PRs are still open on GitHub + error, open_pr_numbers = check_prs_open(synapse.repository, issue.issue_number, synapse.predictions) + if error: + return _reject(error) + + # 3) Validate GitHub identity + account age + github_id, error = validate_github_credentials(uid, synapse.github_access_token) + if error: + return _reject(error) + + # 4) Validate prediction values + error = validate_prediction_values(synapse.predictions) + if error: + return _reject(error) + + # 5) Per-PR cooldown check + submitted_prs = list(synapse.predictions.items()) + for pr_number, pred_value in submitted_prs: + cooldown_remaining = mp_storage.check_cooldown(uid, miner_hotkey, synapse.issue_id, pr_number) + if cooldown_remaining is not None: + return _reject(f'PR #{pr_number} on cooldown ({cooldown_remaining:.0f}s remaining)') + + # 6) Total probability check — exclude all PRs in this submission (they will be overwritten) + submitted_pr_numbers = set(synapse.predictions.keys()) + existing_total = mp_storage.get_miner_total_for_issue( + uid, + miner_hotkey, + synapse.issue_id, + exclude_prs=submitted_pr_numbers, + only_prs=open_pr_numbers, + ) + new_total = sum(synapse.predictions.values()) + if existing_total + new_total > 1.0: + return _reject( + f'Total probability exceeds 1.0 — other open PRs: {existing_total:.4f} + submission: {new_total:.4f} = {existing_total + new_total:.4f}' + ) + + # 7) Compute variance at time of submission and store all predictions + variance = mp_storage.compute_current_variance(synapse.issue_id) + + now = datetime.now(timezone.utc).isoformat() + + for pr_number, pred_value in submitted_prs: + mp_storage.store_prediction( + uid=uid, + hotkey=miner_hotkey, + github_id=github_id, + issue_id=synapse.issue_id, + repository=synapse.repository, + pr_number=pr_number, + prediction=pred_value, + variance_at_prediction=variance, + ) + + # Mirror to Postgres + if validator.db_storage: + validator.db_storage.store_merge_prediction( + uid=uid, + hotkey=miner_hotkey, + github_id=github_id, + issue_id=synapse.issue_id, + repository=synapse.repository, + pr_number=pr_number, + prediction=pred_value, + variance_at_prediction=variance, + timestamp=now, + ) + + bt.logging.success( + f'Merge prediction stored — UID: {uid}, ID: {synapse.issue_id}, ' + f'issue: #{issue.issue_number}, repo: {synapse.repository}, ' + f'PRs: {[pr for pr, _ in submitted_prs]}, github_id: {github_id}' + ) + + synapse.accepted = True + return synapse + + +async def blacklist_prediction(validator: 'Validator', synapse: PredictionSynapse) -> Tuple[bool, str]: + """Reject synapses from unregistered hotkeys.""" + if synapse.dendrite is None or synapse.dendrite.hotkey is None: + return True, 'Missing dendrite or hotkey' + + if synapse.dendrite.hotkey not in validator.metagraph.hotkeys: + return True, 'Unregistered hotkey' + + return False, 'Hotkey recognized' + + +async def priority_prediction(validator: 'Validator', synapse: PredictionSynapse) -> float: + """Priority by stake — higher stake = processed first.""" + if synapse.dendrite is None or synapse.dendrite.hotkey is None: + return 0.0 + + try: + uid = validator.metagraph.hotkeys.index(synapse.dendrite.hotkey) + return float(validator.metagraph.S[uid]) + except ValueError: + return 0.0 diff --git a/gittensor/validator/merge_predictions/mp_storage.py b/gittensor/validator/merge_predictions/mp_storage.py new file mode 100644 index 00000000..0e757980 --- /dev/null +++ b/gittensor/validator/merge_predictions/mp_storage.py @@ -0,0 +1,273 @@ +# Entrius 2025 + +"""SQLite storage for merge predictions. + +Each validator stores predictions independently. One row per miner per PR. +Thread-safe via WAL mode — the axon handler writes while the scoring loop reads. +""" + +import sqlite3 +import threading +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +import bittensor as bt + +from gittensor.constants import PREDICTIONS_COOLDOWN_SECONDS +from gittensor.validator.utils.config import MP_DB_PATH + + +class PredictionStorage: + """Thread-safe SQLite storage for merge predictions.""" + + def __init__(self, db_path: Optional[str] = None): + self._db_path = db_path or MP_DB_PATH + Path(self._db_path).parent.mkdir(parents=True, exist_ok=True) + self._lock = threading.Lock() + self._init_db() + + def _get_connection(self) -> sqlite3.Connection: + conn = sqlite3.connect(self._db_path) + conn.execute('PRAGMA journal_mode=WAL') + conn.execute('PRAGMA busy_timeout=5000') + conn.row_factory = sqlite3.Row + return conn + + def _init_db(self): + with self._get_connection() as conn: + conn.execute(""" + CREATE TABLE IF NOT EXISTS predictions ( + uid INTEGER NOT NULL, + hotkey TEXT NOT NULL, + github_id TEXT NOT NULL, + issue_id INTEGER NOT NULL, + repository TEXT NOT NULL, + pr_number INTEGER NOT NULL, + prediction REAL NOT NULL, + timestamp TEXT NOT NULL, + variance_at_prediction REAL, + PRIMARY KEY (uid, hotkey, github_id, issue_id, pr_number) + ) + """) + conn.execute(""" + CREATE TABLE IF NOT EXISTS prediction_emas ( + github_id TEXT NOT NULL, + ema_score REAL NOT NULL DEFAULT 0.0, + rounds INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL, + PRIMARY KEY (github_id) + ) + """) + conn.execute(""" + CREATE TABLE IF NOT EXISTS settled_issues ( + issue_id INTEGER NOT NULL PRIMARY KEY, + outcome TEXT NOT NULL, + merged_pr_number INTEGER, + settled_at TEXT NOT NULL + ) + """) + conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_predictions_issue + ON predictions (issue_id) + """) + conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_predictions_miner_issue + ON predictions (uid, hotkey, issue_id) + """) + conn.commit() + bt.logging.info(f'Prediction storage initialized at {self._db_path}') + + def check_cooldown(self, uid: int, hotkey: str, issue_id: int, pr_number: int) -> Optional[float]: + """Return seconds remaining on cooldown, or None if no cooldown active.""" + with self._get_connection() as conn: + row = conn.execute( + 'SELECT timestamp FROM predictions WHERE uid = ? AND hotkey = ? AND issue_id = ? AND pr_number = ?', + (uid, hotkey, issue_id, pr_number), + ).fetchone() + + if row is None: + return None + + last_ts = datetime.fromisoformat(row['timestamp']) + elapsed = (datetime.now(timezone.utc) - last_ts).total_seconds() + remaining = PREDICTIONS_COOLDOWN_SECONDS - elapsed + return remaining if remaining > 0 else None + + def get_miner_total_for_issue( + self, + uid: int, + hotkey: str, + issue_id: int, + exclude_prs: Optional[set[int]] = None, + only_prs: Optional[set[int]] = None, + ) -> float: + """Get sum of a miner's existing predictions for an issue. + + Args: + exclude_prs: Exclude these PRs from the sum (for batch updates). + only_prs: If provided, only count predictions on these PRs (open PRs). + Predictions on closed PRs are excluded from the total, + freeing that probability for reallocation. + """ + with self._get_connection() as conn: + query = 'SELECT COALESCE(SUM(prediction), 0.0) as total FROM predictions WHERE uid = ? AND hotkey = ? AND issue_id = ?' + params: list = [uid, hotkey, issue_id] + + if exclude_prs: + placeholders = ','.join('?' for _ in exclude_prs) + query += f' AND pr_number NOT IN ({placeholders})' + params.extend(exclude_prs) + + if only_prs: + placeholders = ','.join('?' for _ in only_prs) + query += f' AND pr_number IN ({placeholders})' + params.extend(only_prs) + + row = conn.execute(query, params).fetchone() + return float(row['total']) + + def compute_current_variance(self, issue_id: int) -> float: + """Compute avg variance across all PRs for an issue (used for consensus bonus).""" + with self._get_connection() as conn: + rows = conn.execute( + """ + SELECT pr_number, AVG(prediction) as mean_pred, + AVG(prediction * prediction) - AVG(prediction) * AVG(prediction) as var_pred + FROM predictions + WHERE issue_id = ? + GROUP BY pr_number + """, + (issue_id,), + ).fetchall() + + if not rows: + return 0.0 + + variances = [max(0.0, float(r['var_pred'])) for r in rows] + return sum(variances) / len(variances) + + def store_prediction( + self, + uid: int, + hotkey: str, + github_id: str, + issue_id: int, + repository: str, + pr_number: int, + prediction: float, + variance_at_prediction: float, + ) -> None: + """Insert or replace a single PR prediction. Resets timestamp on that PR only.""" + now = datetime.now(timezone.utc).isoformat() + + with self._lock: + with self._get_connection() as conn: + conn.execute( + """ + INSERT INTO predictions (uid, hotkey, github_id, issue_id, repository, pr_number, prediction, timestamp, variance_at_prediction) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (uid, hotkey, github_id, issue_id, pr_number) + DO UPDATE SET prediction = excluded.prediction, + timestamp = excluded.timestamp, + variance_at_prediction = excluded.variance_at_prediction + """, + (uid, hotkey, github_id, issue_id, repository, pr_number, prediction, now, variance_at_prediction), + ) + conn.commit() + + def get_peak_variance_time(self, issue_id: int) -> Optional[datetime]: + """Get the timestamp when variance was highest for an issue. + + Returns the prediction timestamp with the max variance_at_prediction, + or None if no predictions exist. + """ + with self._get_connection() as conn: + row = conn.execute( + 'SELECT timestamp FROM predictions WHERE issue_id = ? ORDER BY variance_at_prediction DESC LIMIT 1', + (issue_id,), + ).fetchone() + if row is None: + return None + return datetime.fromisoformat(row['timestamp']) + + def get_predictions_for_issue(self, issue_id: int) -> list[dict]: + """Get all predictions for an issue (used at settlement).""" + with self._get_connection() as conn: + rows = conn.execute( + 'SELECT * FROM predictions WHERE issue_id = ? ORDER BY uid, pr_number', + (issue_id,), + ).fetchall() + return [dict(r) for r in rows] + + def delete_predictions_for_issue(self, issue_id: int) -> int: + """Delete all predictions for a settled/cancelled issue. Returns rows deleted.""" + with self._lock: + with self._get_connection() as conn: + cursor = conn.execute('DELETE FROM predictions WHERE issue_id = ?', (issue_id,)) + conn.commit() + return cursor.rowcount + + # ========================================================================= + # Settlement tracking + # ========================================================================= + + def is_issue_settled(self, issue_id: int) -> bool: + """Check if an issue has already been settled.""" + with self._get_connection() as conn: + row = conn.execute( + 'SELECT 1 FROM settled_issues WHERE issue_id = ?', + (issue_id,), + ).fetchone() + return row is not None + + def mark_issue_settled(self, issue_id: int, outcome: str, merged_pr_number: int | None = None) -> None: + """Record that an issue has been settled. Idempotent (INSERT OR IGNORE).""" + now = datetime.now(timezone.utc).isoformat() + with self._lock: + with self._get_connection() as conn: + conn.execute( + 'INSERT OR IGNORE INTO settled_issues (issue_id, outcome, merged_pr_number, settled_at) ' + 'VALUES (?, ?, ?, ?)', + (issue_id, outcome, merged_pr_number, now), + ) + conn.commit() + + # ========================================================================= + # EMA tracking + # ========================================================================= + + def get_ema(self, github_id: str) -> float: + """Get a miner's current prediction EMA score. Returns 0.0 if no record.""" + with self._get_connection() as conn: + row = conn.execute( + 'SELECT ema_score FROM prediction_emas WHERE github_id = ?', + (github_id,), + ).fetchone() + return float(row['ema_score']) if row else 0.0 + + def update_ema(self, github_id: str, new_ema: float) -> None: + """Upsert a miner's prediction EMA score, keyed by github_id.""" + now = datetime.now(timezone.utc).isoformat() + with self._lock: + with self._get_connection() as conn: + conn.execute( + """ + INSERT INTO prediction_emas (github_id, ema_score, rounds, updated_at) + VALUES (?, ?, 1, ?) + ON CONFLICT (github_id) + DO UPDATE SET ema_score = excluded.ema_score, + rounds = prediction_emas.rounds + 1, + updated_at = excluded.updated_at + """, + (github_id, new_ema, now), + ) + conn.commit() + + def get_all_emas(self) -> list[dict]: + """Get all miner EMA scores. Used at weight-setting time for blending.""" + with self._get_connection() as conn: + rows = conn.execute( + 'SELECT github_id, ema_score, rounds, updated_at FROM prediction_emas ORDER BY github_id', + ).fetchall() + return [dict(r) for r in rows] diff --git a/gittensor/validator/merge_predictions/scoring.py b/gittensor/validator/merge_predictions/scoring.py new file mode 100644 index 00000000..eb8a7c7e --- /dev/null +++ b/gittensor/validator/merge_predictions/scoring.py @@ -0,0 +1,245 @@ +# Entrius 2025 + +"""Pure scoring functions for merge predictions. + +All functions are stateless — data in, scores out. No DB queries or side effects. + +Formula per PR: + pr_score = correctness³ * (1 + timeliness_bonus + consensus_bonus + order_bonus) + +Where: + - correctness: log-loss derived (prediction for merged, 1-prediction for non-merged), cubed + - timeliness_bonus: 0.0-0.75, rewards early predictions (gated: raw correctness >= 0.66) + - consensus_bonus: 0.0-0.25, rewards pre-convergence predictions (gated: raw correctness >= 0.66) + - order_bonus: 0.0-0.75, rewards first correct predictor, merged PR only (gated: raw correctness >= 0.66) + +All bonuses require raw correctness >= ORDER_CORRECTNESS_THRESHOLD to activate. +Issue score: weighted_mean × coverage, where merged PR gets weight=N, non-merged weight=1, + and coverage = prs_predicted / total_prs. +""" + +from dataclasses import dataclass +from datetime import datetime + +from gittensor.constants import ( + PREDICTIONS_CORRECTNESS_EXPONENT, + PREDICTIONS_EMA_BETA, + PREDICTIONS_MAX_CONSENSUS_BONUS, + PREDICTIONS_MAX_ORDER_BONUS, + PREDICTIONS_MAX_TIMELINESS_BONUS, + PREDICTIONS_ORDER_CORRECTNESS_THRESHOLD, + PREDICTIONS_TIMELINESS_EXPONENT, +) + +# ============================================================================= +# Data structures +# ============================================================================= + + +@dataclass +class PrPrediction: + pr_number: int + prediction: float # 0.0-1.0 + prediction_time: datetime # when this PR prediction was submitted + variance_at_prediction: float + + +@dataclass +class PrOutcome: + pr_number: int + outcome: float # 1.0 for merged PR, 0.0 for all others + pr_open_time: datetime # when this PR was opened on GitHub + + +@dataclass +class PrScore: + pr_number: int + correctness: float + timeliness_bonus: float + consensus_bonus: float + order_bonus: float + score: float # correctness³ * (1 + timeliness + consensus + order) + + +@dataclass +class MinerIssueScore: + uid: int + pr_scores: list[PrScore] + issue_score: float # weighted mean (merged PR weight=N, non-merged weight=1) + + +# ============================================================================= +# Scoring functions +# ============================================================================= + + +def raw_correctness(prediction: float, outcome: float) -> float: + """Log-loss derived correctness before exponentiation. + + Merged PR (outcome=1.0): score = prediction. + Non-merged PR (outcome=0.0): score = 1 - prediction. + """ + return prediction if outcome == 1.0 else 1.0 - prediction + + +def score_correctness(prediction: float, outcome: float) -> float: + """Cubed correctness. Heavily punishes inaccuracy.""" + return raw_correctness(prediction, outcome) ** PREDICTIONS_CORRECTNESS_EXPONENT + + +def score_timeliness(prediction_time: datetime, settlement_time: datetime, pr_open_time: datetime) -> float: + """Bounded timeliness bonus (0.0 to MAX_TIMELINESS_BONUS). + + Rewards earlier predictions within the PR's lifetime window. + """ + total_window = (settlement_time - pr_open_time).total_seconds() + if total_window <= 0: + return 0.0 + + time_remaining = (settlement_time - prediction_time).total_seconds() + ratio = max(0.0, min(1.0, time_remaining / total_window)) + return PREDICTIONS_MAX_TIMELINESS_BONUS * ratio**PREDICTIONS_TIMELINESS_EXPONENT + + +def score_consensus_bonus(prediction_time: datetime, peak_variance_time: datetime, settlement_time: datetime) -> float: + """Bounded consensus bonus (0.0 to MAX_CONSENSUS_BONUS). + + Rewards predictions made before or near peak disagreement. + Pre-peak: full bonus. Post-peak: linearly decays to 0 at settlement. + """ + if prediction_time <= peak_variance_time: + return PREDICTIONS_MAX_CONSENSUS_BONUS + + remaining_window = (settlement_time - peak_variance_time).total_seconds() + if remaining_window <= 0: + return 0.0 + + time_after_peak = (prediction_time - peak_variance_time).total_seconds() + ratio = max(0.0, min(1.0, time_after_peak / remaining_window)) + return PREDICTIONS_MAX_CONSENSUS_BONUS * (1.0 - ratio) + + +def score_order_bonus(rank: int) -> float: + """Order bonus for the merged PR only. bonus = max / rank. + + Rank 0 means unqualified (below correctness threshold). Returns 0.0. + """ + if rank <= 0: + return 0.0 + return PREDICTIONS_MAX_ORDER_BONUS / rank + + +# ============================================================================= +# Order ranking (cross-miner) +# ============================================================================= + + +def compute_merged_pr_order_ranks( + all_miners_predictions: dict[int, list[PrPrediction]], + merged_pr_number: int, +) -> dict[int, int]: + """Rank miners by who first correctly predicted the merged PR. + + Only miners with raw correctness >= threshold qualify. + Ranked by prediction_time (earliest first). + + Returns: + dict mapping uid -> rank (1-indexed). Unqualified miners are absent. + """ + qualifying = [] + + for uid, predictions in all_miners_predictions.items(): + for pred in predictions: + if pred.pr_number != merged_pr_number: + continue + rc = raw_correctness(pred.prediction, 1.0) + if rc >= PREDICTIONS_ORDER_CORRECTNESS_THRESHOLD: + qualifying.append((uid, pred.prediction_time)) + break + + qualifying.sort(key=lambda x: x[1]) + + return {uid: rank for rank, (uid, _) in enumerate(qualifying, start=1)} + + +# ============================================================================= +# Aggregation +# ============================================================================= + + +def score_miner_issue( + uid: int, + predictions: list[PrPrediction], + outcomes: list[PrOutcome], + settlement_time: datetime, + peak_variance_time: datetime, + merged_pr_order_ranks: dict[int, int], +) -> MinerIssueScore: + """Score a single miner's predictions for one issue. + + Fills unpredicted PRs, scores each PR, then computes a weighted issue score + where the merged PR gets weight=N (total PRs) and non-merged get weight=1. + """ + outcome_map = {o.pr_number: o for o in outcomes} + merged_prs = {o.pr_number for o in outcomes if o.outcome == 1.0} + n_prs = len(outcomes) + + miner_rank = merged_pr_order_ranks.get(uid, 0) + + pr_scores = [] + for pred in predictions: + outcome = outcome_map.get(pred.pr_number) + if outcome is None: + continue + + rc = raw_correctness(pred.prediction, outcome.outcome) + correctness = rc**PREDICTIONS_CORRECTNESS_EXPONENT + qualifies_for_bonus = rc >= PREDICTIONS_ORDER_CORRECTNESS_THRESHOLD + + timeliness_bonus = ( + score_timeliness(pred.prediction_time, settlement_time, outcome.pr_open_time) + if qualifies_for_bonus + else 0.0 + ) + consensus_bonus = ( + score_consensus_bonus(pred.prediction_time, peak_variance_time, settlement_time) + if qualifies_for_bonus + else 0.0 + ) + + is_merged = pred.pr_number in merged_prs + order_bonus = score_order_bonus(miner_rank) if is_merged and qualifies_for_bonus else 0.0 + + score = correctness * (1.0 + timeliness_bonus + consensus_bonus + order_bonus) + pr_scores.append( + PrScore( + pr_number=pred.pr_number, + correctness=correctness, + timeliness_bonus=timeliness_bonus, + consensus_bonus=consensus_bonus, + order_bonus=order_bonus, + score=score, + ) + ) + + # Weighted mean: merged PR gets weight=N, non-merged get weight=1 + total_weight = 0.0 + weighted_sum = 0.0 + for ps in pr_scores: + weight = n_prs if ps.pr_number in merged_prs else 1.0 + weighted_sum += ps.score * weight + total_weight += weight + + raw_issue_score = weighted_sum / total_weight if total_weight > 0 else 0.0 + + # Coverage multiplier: reward miners who reviewed the full field + prs_predicted = len(pr_scores) + coverage = prs_predicted / n_prs if n_prs > 0 else 0.0 + issue_score = raw_issue_score * coverage + + return MinerIssueScore(uid=uid, pr_scores=pr_scores, issue_score=issue_score) + + +def update_ema(current_round_score: float, previous_ema: float) -> float: + """Exponential moving average for a miner's prediction track record.""" + return PREDICTIONS_EMA_BETA * current_round_score + (1.0 - PREDICTIONS_EMA_BETA) * previous_ema diff --git a/gittensor/validator/merge_predictions/settlement.py b/gittensor/validator/merge_predictions/settlement.py new file mode 100644 index 00000000..86e233e5 --- /dev/null +++ b/gittensor/validator/merge_predictions/settlement.py @@ -0,0 +1,411 @@ +# Entrius 2025 + +"""Settlement orchestrator for merge predictions. + +Queries COMPLETED and CANCELLED issues from the smart contract and scores +miners' predictions, updating their EMA. + +- COMPLETED issues: scored normally, predictions deleted after settlement. +- CANCELLED issues with a merged PR: scored (solver wasn't an eligible miner, + but the PR was still merged — predictions are still valid). +- CANCELLED issues without a merged PR: voided — predictions deleted, no EMA impact. + +The `settled_issues` table is the durable settled marker — once an issue is +recorded there, subsequent passes skip it regardless of prediction state. +""" + +from collections import defaultdict +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Dict + +import bittensor as bt + +from gittensor.classes import MinerEvaluation +from gittensor.utils.github_api_tools import check_github_issue_closed, get_pr_open_times +from gittensor.validator.issue_competitions.contract_client import IssueCompetitionContractClient, IssueStatus +from gittensor.validator.merge_predictions.scoring import ( + MinerIssueScore, + PrOutcome, + PrPrediction, + compute_merged_pr_order_ranks, + score_miner_issue, + update_ema, +) +from gittensor.validator.utils.config import GITTENSOR_VALIDATOR_PAT +from gittensor.validator.utils.issue_competitions import get_contract_address + +if TYPE_CHECKING: + from neurons.validator import Validator + + +# ============================================================================= +# Helper functions +# ============================================================================= + + +def db_storage_void(validator: 'Validator', issue_id: int) -> None: + """Best-effort mirror of a voided issue to Postgres.""" + if validator.db_storage: + now = datetime.now(timezone.utc).isoformat() + validator.db_storage.store_merge_settled_issue(issue_id, 'voided', None, now) + + +def _build_outcomes( + predictions: list[dict], + merged_pr_number: int, + repository: str, + pr_open_times: dict[int, datetime], + settlement_time: datetime, +) -> list[PrOutcome]: + """Build PrOutcome list from raw prediction rows + merged PR number.""" + predicted_pr_numbers = list({p['pr_number'] for p in predictions}) + + if merged_pr_number not in predicted_pr_numbers: + predicted_pr_numbers.append(merged_pr_number) + + outcomes: list[PrOutcome] = [] + for pr_num in predicted_pr_numbers: + outcome_value = 1.0 if pr_num == merged_pr_number else 0.0 + pr_open_time = pr_open_times.get(pr_num) + if pr_open_time is None: + pr_pred_times = [datetime.fromisoformat(p['timestamp']) for p in predictions if p['pr_number'] == pr_num] + pr_open_time = min(pr_pred_times) if pr_pred_times else settlement_time + + outcomes.append(PrOutcome(pr_number=pr_num, outcome=outcome_value, pr_open_time=pr_open_time)) + + return outcomes + + +def _group_miner_predictions( + predictions: list[dict], + metagraph, +) -> tuple[dict[int, list[PrPrediction]], dict[int, str]]: + """Filter deregistered miners and group predictions by UID. + + Returns: + (all_miners_predictions, uid_to_github_id) + """ + all_miners_predictions: dict[int, list[PrPrediction]] = defaultdict(list) + uid_to_github_id: dict[int, str] = {} + + for p in predictions: + uid = p['uid'] + if uid >= len(metagraph.hotkeys) or metagraph.hotkeys[uid] != p['hotkey']: + bt.logging.debug(f'Merge predictions: skipping deregistered miner uid={uid} hotkey={p["hotkey"][:12]}...') + continue + + all_miners_predictions[uid].append( + PrPrediction( + pr_number=p['pr_number'], + prediction=p['prediction'], + prediction_time=datetime.fromisoformat(p['timestamp']), + variance_at_prediction=p.get('variance_at_prediction', 0.0) or 0.0, + ) + ) + uid_to_github_id[uid] = p['github_id'] + + return dict(all_miners_predictions), uid_to_github_id + + +def _score_and_update_emas( + validator: 'Validator', + miners_preds: dict[int, list[PrPrediction]], + uid_to_github_id: dict[int, str], + outcomes: list[PrOutcome], + settlement_time: datetime, + peak_variance_time: datetime, + order_ranks: dict[int, int], +) -> list[dict]: + """Score each miner and update EMA. Returns list of result dicts for logging.""" + mp_storage = validator.mp_storage + results = [] + + for uid, miner_preds in miners_preds.items(): + github_id = uid_to_github_id.get(uid) + if not github_id: + bt.logging.warning(f'Merge predictions: no github_id for uid={uid}, skipping EMA update') + continue + + issue_score: MinerIssueScore = score_miner_issue( + uid=uid, + predictions=miner_preds, + outcomes=outcomes, + settlement_time=settlement_time, + peak_variance_time=peak_variance_time, + merged_pr_order_ranks=order_ranks, + ) + + previous_ema = mp_storage.get_ema(github_id) + new_ema = update_ema(issue_score.issue_score, previous_ema) + mp_storage.update_ema(github_id, new_ema) + + # Mirror EMA to Postgres + if validator.db_storage: + now = datetime.now(timezone.utc).isoformat() + validator.db_storage.store_merge_prediction_ema(github_id, new_ema, 1, now) + + results.append( + { + 'uid': uid, + 'github_id': github_id, + 'score': issue_score.issue_score, + 'previous_ema': previous_ema, + 'new_ema': new_ema, + 'rank': order_ranks.get(uid, 0), + 'prs_predicted': len(miner_preds), + } + ) + + return results + + +def _log_issue_settlement( + issue_label: str, + merged_pr_number: int, + all_miners_predictions: dict[int, list[PrPrediction]], + uid_to_github_id: dict[int, str], + miner_results: list[dict], +) -> None: + """Rich per-issue logging block.""" + # Submission summary + total_submissions = sum(len(preds) for preds in all_miners_predictions.values()) + bt.logging.info(f' {total_submissions} submissions from {len(all_miners_predictions)} miners:') + + for uid, preds in all_miners_predictions.items(): + gh_id = uid_to_github_id.get(uid, '?') + merged_preds = [p for p in preds if p.pr_number == merged_pr_number] + avg_on_merged = sum(p.prediction for p in merged_preds) / len(merged_preds) if merged_preds else 0.0 + bt.logging.info( + f' UID: {uid} (gh: {gh_id}) PRs predicted: {len(preds)} ' + f'avg on merged PR #{merged_pr_number}: {avg_on_merged:.2f}' + ) + + # Scoring results + if miner_results: + sorted_results = sorted(miner_results, key=lambda r: r['score'], reverse=True) + bt.logging.info(' Scoring results:') + for r in sorted_results: + rank_str = f'rank #{r["rank"]}' if r['rank'] > 0 else 'unranked' + marker = '\u2605' if r == sorted_results[0] else ' ' + bt.logging.info( + f' {marker} UID: {r["uid"]} score: {r["score"]:.4f} ' + f'ema: {r["previous_ema"]:.4f} \u2192 {r["new_ema"]:.4f} ({rank_str})' + ) + + +def _settle_issue( + validator: 'Validator', + issue, + issue_label: str, + merged_pr_number: int, + settlement_reason: str = 'completed', +) -> bool: + """Full settlement pipeline for one issue. + + Loads predictions, builds outcomes, scores, updates EMAs, logs, deletes. + Shared by both COMPLETED and CANCELLED-with-merge paths. + + Returns True if settled successfully. + """ + mp_storage = validator.mp_storage + + predictions = mp_storage.get_predictions_for_issue(issue.id) + if not predictions: + return False + + unique_prs = {p['pr_number'] for p in predictions} + bt.logging.info( + f'--- Settling {settlement_reason} issue ID: {issue.id}, ' + f'{issue.repository_full_name}#{issue.issue_number}, ' + f'{len(unique_prs)} PRs submitted (merged PR #{merged_pr_number}) ---' + ) + + settlement_time = datetime.now(timezone.utc) + + peak_variance_time = mp_storage.get_peak_variance_time(issue.id) + if peak_variance_time is None: + peak_variance_time = settlement_time + + # Collect unique PR numbers for open-time lookup + predicted_pr_numbers = list({p['pr_number'] for p in predictions}) + if merged_pr_number not in predicted_pr_numbers: + predicted_pr_numbers.append(merged_pr_number) + + pr_open_times = get_pr_open_times(issue.repository_full_name, predicted_pr_numbers, GITTENSOR_VALIDATOR_PAT) + + outcomes = _build_outcomes( + predictions, merged_pr_number, issue.repository_full_name, pr_open_times, settlement_time + ) + all_miners_predictions, uid_to_github_id = _group_miner_predictions(predictions, validator.metagraph) + + if not all_miners_predictions: + bt.logging.debug(f'Merge predictions: no active miners had predictions for {issue_label}') + rows_deleted = mp_storage.delete_predictions_for_issue(issue.id) + bt.logging.info(f' Predictions deleted ({rows_deleted} rows)') + return False + + order_ranks = compute_merged_pr_order_ranks(all_miners_predictions, merged_pr_number) + + miner_results = _score_and_update_emas( + validator, + all_miners_predictions, + uid_to_github_id, + outcomes, + settlement_time, + peak_variance_time, + order_ranks, + ) + + _log_issue_settlement(issue_label, merged_pr_number, all_miners_predictions, uid_to_github_id, miner_results) + + rows_deleted = mp_storage.delete_predictions_for_issue(issue.id) + bt.logging.info(f' Predictions deleted ({rows_deleted} rows)') + + mp_storage.mark_issue_settled(issue.id, 'scored', merged_pr_number) + + # Mirror settlement + delete to Postgres + if validator.db_storage: + now = datetime.now(timezone.utc).isoformat() + validator.db_storage.store_merge_settled_issue(issue.id, 'scored', merged_pr_number, now) + + return True + + +# ============================================================================= +# Main settlement function +# ============================================================================= + + +async def merge_predictions( + self: 'Validator', + miner_evaluations: Dict[int, MinerEvaluation], +) -> None: + """Settle merge predictions for COMPLETED and CANCELLED issues. + + 1. Query COMPLETED issues from contract + - Skip if already in settled_issues table + - check_github_issue_closed to get merged PR number + - Score miners, update EMAs, delete predictions, record in settled_issues + + 2. Query CANCELLED issues from contract + - Skip if already in settled_issues table + - check_github_issue_closed to determine WHY it was cancelled: + a) Merged PR exists -> score + delete + record as 'scored' + b) No merged PR -> void: delete predictions + record as 'voided', no EMA impact + """ + try: + if not GITTENSOR_VALIDATOR_PAT: + bt.logging.warning( + 'GITTENSOR_VALIDATOR_PAT not set, skipping merge predictions settlement. (This DOES affect vstrust/consensus)' + ) + return + + contract_addr = get_contract_address() + if not contract_addr: + bt.logging.warning('Merge predictions: no contract address configured') + return + + bt.logging.info('***** Starting Merge Predictions Settlement *****') + + contract_client = IssueCompetitionContractClient( + contract_address=contract_addr, + subtensor=self.subtensor, + ) + + completed_settled = 0 + cancelled_settled = 0 + voided = 0 + skipped = 0 + + # --- COMPLETED issues --- + completed_issues = contract_client.get_issues_by_status(IssueStatus.COMPLETED) + bt.logging.info(f'Merge predictions: {len(completed_issues)} completed issues to check') + + for issue in completed_issues: + issue_label = f'{issue.repository_full_name}#{issue.issue_number} (id={issue.id})' + try: + if self.mp_storage.is_issue_settled(issue.id): + skipped += 1 + continue + + github_state = check_github_issue_closed( + issue.repository_full_name, issue.issue_number, GITTENSOR_VALIDATOR_PAT + ) + + if github_state is None: + bt.logging.debug(f'Merge predictions: could not check GitHub state for {issue_label}') + continue + + merged_pr_number = github_state.get('pr_number') + if not merged_pr_number: + bt.logging.warning( + f'Merge predictions: completed issue {issue_label} has no merged PR on GitHub, voiding' + ) + rows_deleted = self.mp_storage.delete_predictions_for_issue(issue.id) + bt.logging.info( + f' Voiding completed issue {issue_label} — no merged PR found, ' + f'{rows_deleted} predictions deleted, no EMA impact' + ) + self.mp_storage.mark_issue_settled(issue.id, 'voided') + db_storage_void(self, issue.id) + voided += 1 + continue + + if _settle_issue(self, issue, issue_label, merged_pr_number): + completed_settled += 1 + else: + skipped += 1 + + except Exception as e: + bt.logging.error(f'Merge predictions: error processing completed {issue_label}: {e}') + + # --- CANCELLED issues --- + cancelled_issues = contract_client.get_issues_by_status(IssueStatus.CANCELLED) + bt.logging.info(f'Merge predictions: {len(cancelled_issues)} cancelled issues to check') + + for issue in cancelled_issues: + issue_label = f'{issue.repository_full_name}#{issue.issue_number} (id={issue.id})' + try: + if self.mp_storage.is_issue_settled(issue.id): + skipped += 1 + continue + + github_state = check_github_issue_closed( + issue.repository_full_name, issue.issue_number, GITTENSOR_VALIDATOR_PAT + ) + + if github_state is None: + bt.logging.debug(f'Merge predictions: could not check GitHub state for {issue_label}') + continue + + merged_pr_number = github_state.get('pr_number') + + if merged_pr_number: + # Cancelled but PR was merged (solver not in subnet) — still score + if _settle_issue(self, issue, issue_label, merged_pr_number, settlement_reason='cancelled'): + cancelled_settled += 1 + else: + skipped += 1 + else: + # No merged PR — void: delete predictions, no EMA impact + rows_deleted = self.mp_storage.delete_predictions_for_issue(issue.id) + bt.logging.info( + f' Voiding cancelled issue ID {issue.id}, {issue.repository_full_name}' + f'#{issue.issue_number} — closed without merge, ' + f'{rows_deleted} predictions deleted, no EMA impact' + ) + self.mp_storage.mark_issue_settled(issue.id, 'voided') + db_storage_void(self, issue.id) + voided += 1 + + except Exception as e: + bt.logging.error(f'Merge predictions: error processing cancelled {issue_label}: {e}') + + bt.logging.info( + f'***** Merge Predictions Settlement Complete: ' + f'{completed_settled} completed settled, {cancelled_settled} cancelled settled, ' + f'{voided} voided, {skipped} skipped *****' + ) + + except Exception as e: + bt.logging.error(f'Merge predictions settlement failed: {e}') diff --git a/gittensor/validator/merge_predictions/validation.py b/gittensor/validator/merge_predictions/validation.py new file mode 100644 index 00000000..c6de5f25 --- /dev/null +++ b/gittensor/validator/merge_predictions/validation.py @@ -0,0 +1,26 @@ +# Entrius 2025 + +"""Pure input validation for prediction payloads.""" + +from gittensor.constants import ( + PREDICTIONS_MAX_VALUE, + PREDICTIONS_MIN_VALUE, +) + + +def validate_prediction_values(predictions: dict[int, float]) -> str | None: + """Validate prediction payload structure and values.""" + if not predictions: + return 'Empty predictions' + + for pr_number, value in predictions.items(): + if not isinstance(pr_number, int) or pr_number <= 0: + return f'Invalid PR number: {pr_number}' + if not (PREDICTIONS_MIN_VALUE <= value <= PREDICTIONS_MAX_VALUE): + return f'Prediction for PR #{pr_number} out of range: {value} (must be {PREDICTIONS_MIN_VALUE}-{PREDICTIONS_MAX_VALUE})' + + total = sum(predictions.values()) + if total > 1.0: + return f'Submission total exceeds 1.0: {total:.4f}' + + return None diff --git a/gittensor/validator/evaluation/credibility.py b/gittensor/validator/oss_contributions/credibility.py similarity index 99% rename from gittensor/validator/evaluation/credibility.py rename to gittensor/validator/oss_contributions/credibility.py index 3f0707a8..faef444b 100644 --- a/gittensor/validator/evaluation/credibility.py +++ b/gittensor/validator/oss_contributions/credibility.py @@ -5,7 +5,7 @@ import bittensor as bt -from gittensor.validator.configurations.tier_config import ( +from gittensor.validator.oss_contributions.tier_config import ( TIERS, TIERS_ORDER, Tier, diff --git a/gittensor/validator/evaluation/dynamic_emissions.py b/gittensor/validator/oss_contributions/dynamic_emissions.py similarity index 100% rename from gittensor/validator/evaluation/dynamic_emissions.py rename to gittensor/validator/oss_contributions/dynamic_emissions.py diff --git a/gittensor/validator/evaluation/inspections.py b/gittensor/validator/oss_contributions/inspections.py similarity index 74% rename from gittensor/validator/evaluation/inspections.py rename to gittensor/validator/oss_contributions/inspections.py index 489136d5..7cb90626 100644 --- a/gittensor/validator/evaluation/inspections.py +++ b/gittensor/validator/oss_contributions/inspections.py @@ -2,20 +2,14 @@ # Copyright © 2025 Entrius from collections import defaultdict -from typing import Dict, List, Optional, Tuple +from typing import Dict, List import bittensor as bt from gittensor.classes import MinerEvaluation -from gittensor.constants import ( - MIN_GITHUB_ACCOUNT_AGE, - RECYCLE_UID, -) +from gittensor.constants import RECYCLE_UID from gittensor.synapses import GitPatSynapse -from gittensor.utils.github_api_tools import ( - get_github_account_age_days, - get_github_id, -) +from gittensor.validator.utils.github_validation import validate_github_credentials def detect_and_penalize_miners_sharing_github(miner_evaluations: Dict[int, MinerEvaluation]): @@ -80,21 +74,3 @@ def validate_response_and_initialize_miner_evaluation(uid: int, response: GitPat miner_eval.github_id = github_id miner_eval.github_pat = response.github_access_token return miner_eval - - -def validate_github_credentials(uid: int, pat: Optional[str]) -> Tuple[Optional[str], Optional[str]]: - """Validate PAT and return (github_id, error_reason) tuple.""" - if not pat: - return None, f'No Github PAT provided by miner {uid}' - - github_id = get_github_id(pat) - if not github_id: - return None, f"No Github id found for miner {uid}'s PAT" - - account_age = get_github_account_age_days(pat) - if not account_age: - return None, f'Could not determine Github account age for miner {uid}' - if account_age < MIN_GITHUB_ACCOUNT_AGE: - return None, f"Miner {uid}'s Github account too young ({account_age} < {MIN_GITHUB_ACCOUNT_AGE} days)" - - return github_id, None diff --git a/gittensor/validator/evaluation/normalize.py b/gittensor/validator/oss_contributions/normalize.py similarity index 100% rename from gittensor/validator/evaluation/normalize.py rename to gittensor/validator/oss_contributions/normalize.py diff --git a/gittensor/validator/evaluation/reward.py b/gittensor/validator/oss_contributions/reward.py similarity index 92% rename from gittensor/validator/evaluation/reward.py rename to gittensor/validator/oss_contributions/reward.py index 62ad3455..0b15ef3e 100644 --- a/gittensor/validator/evaluation/reward.py +++ b/gittensor/validator/oss_contributions/reward.py @@ -11,17 +11,17 @@ from gittensor.classes import MinerEvaluation from gittensor.synapses import GitPatSynapse from gittensor.utils.github_api_tools import load_miners_prs -from gittensor.validator.evaluation.dynamic_emissions import apply_dynamic_emissions_using_network_contributions -from gittensor.validator.evaluation.inspections import ( +from gittensor.validator.oss_contributions.dynamic_emissions import apply_dynamic_emissions_using_network_contributions +from gittensor.validator.oss_contributions.inspections import ( detect_and_penalize_miners_sharing_github, validate_response_and_initialize_miner_evaluation, ) -from gittensor.validator.evaluation.normalize import normalize_rewards_linear -from gittensor.validator.evaluation.scoring import ( +from gittensor.validator.oss_contributions.normalize import normalize_rewards_linear +from gittensor.validator.oss_contributions.scoring import ( finalize_miner_scores, score_miner_prs, ) -from gittensor.validator.evaluation.tier_emissions import allocate_emissions_by_tier +from gittensor.validator.oss_contributions.tier_config import allocate_emissions_by_tier from gittensor.validator.utils.load_weights import LanguageConfig, RepositoryConfig, TokenConfig # NOTE: there was a circular import error, needed this if to resolve it diff --git a/gittensor/validator/evaluation/scoring.py b/gittensor/validator/oss_contributions/scoring.py similarity index 99% rename from gittensor/validator/evaluation/scoring.py rename to gittensor/validator/oss_contributions/scoring.py index fb313ffe..09d57888 100644 --- a/gittensor/validator/evaluation/scoring.py +++ b/gittensor/validator/oss_contributions/scoring.py @@ -36,7 +36,12 @@ fetch_file_contents_with_base, get_pull_request_file_changes, ) -from gittensor.validator.configurations.tier_config import ( +from gittensor.validator.oss_contributions.credibility import ( + calculate_credibility_per_tier, + calculate_tier_stats, + is_tier_unlocked, +) +from gittensor.validator.oss_contributions.tier_config import ( TIERS, TIERS_ORDER, Tier, @@ -44,11 +49,6 @@ TierStats, get_tier_from_config, ) -from gittensor.validator.evaluation.credibility import ( - calculate_credibility_per_tier, - calculate_tier_stats, - is_tier_unlocked, -) from gittensor.validator.utils.load_weights import LanguageConfig, RepositoryConfig, TokenConfig from gittensor.validator.utils.tree_sitter_scoring import calculate_token_score_from_file_changes diff --git a/gittensor/validator/evaluation/tier_emissions.py b/gittensor/validator/oss_contributions/tier_config.py similarity index 53% rename from gittensor/validator/evaluation/tier_emissions.py rename to gittensor/validator/oss_contributions/tier_config.py index fe2b97f4..5da7ca8e 100644 --- a/gittensor/validator/evaluation/tier_emissions.py +++ b/gittensor/validator/oss_contributions/tier_config.py @@ -1,20 +1,131 @@ -# The MIT License (MIT) -# Copyright © 2025 Entrius +from __future__ import annotations -""" -Tier-based emission allocation module. +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING, Dict, Optional -Implements fixed tier-based emission splits (Bronze 15%, Silver 35%, Gold 50%) -so miners compete for rewards within their tier rather than globally. -""" +import bittensor as bt -from typing import Dict +from gittensor.constants import ( + DEFAULT_COLLATERAL_PERCENT, + DEFAULT_MAX_CONTRIBUTION_SCORE_FOR_FULL_BONUS, + DEFAULT_MERGED_PR_BASE_SCORE, + MAX_CONTRIBUTION_BONUS, + TIER_EMISSION_SPLITS, +) -import bittensor as bt +if TYPE_CHECKING: + from gittensor.classes import MinerEvaluation + + +@dataclass +class TierStats: + """Statistics for a single tier.""" + + merged_count: int = 0 + closed_count: int = 0 + open_count: int = 0 + + unique_repo_contribution_count: int = 0 + # Unique repos that meet a min token score threshold + qualified_unique_repo_count: int = 0 + + # Included as scoring details at the tier level + earned_score: float = 0.0 + collateral_score: float = 0.0 + + # Token scoring breakdown for this tier + token_score: float = 0.0 + structural_count: int = 0 + structural_score: float = 0.0 + leaf_count: int = 0 + leaf_score: float = 0.0 + + @property + def total_attempts(self) -> int: + return self.merged_count + self.closed_count + + @property + def total_prs(self) -> int: + return self.merged_count + self.closed_count + self.open_count + + @property + def credibility(self) -> float: + return self.merged_count / self.total_attempts if self.total_attempts > 0 else 0.0 + + +class Tier(str, Enum): + BRONZE = 'Bronze' + SILVER = 'Silver' + GOLD = 'Gold' -from gittensor.classes import MinerEvaluation -from gittensor.constants import TIER_EMISSION_SPLITS -from gittensor.validator.configurations.tier_config import TIERS_ORDER, Tier + +TIER_DEFAULTS = { + 'merged_pr_base_score': DEFAULT_MERGED_PR_BASE_SCORE, + 'contribution_score_for_full_bonus': DEFAULT_MAX_CONTRIBUTION_SCORE_FOR_FULL_BONUS, + 'contribution_score_max_bonus': MAX_CONTRIBUTION_BONUS, + 'open_pr_collateral_percentage': DEFAULT_COLLATERAL_PERCENT, +} + + +@dataclass(frozen=True) +class TierConfig: + required_credibility: Optional[float] + required_min_token_score: Optional[float] # Minimum total token score to unlock tier + # Unique repos with min token score requirement (both must be set or both None) + required_unique_repos_count: Optional[int] # Number of unique repos needed + required_min_token_score_per_repo: Optional[float] # Min token score each repo must have + + # Tier-specific scaling + credibility_scalar: int + + # Defaults (can override per-tier if needed) + merged_pr_base_score: int = TIER_DEFAULTS['merged_pr_base_score'] + contribution_score_for_full_bonus: int = TIER_DEFAULTS['contribution_score_for_full_bonus'] + contribution_score_max_bonus: int = TIER_DEFAULTS['contribution_score_max_bonus'] + open_pr_collateral_percentage: int = TIER_DEFAULTS['open_pr_collateral_percentage'] + + +TIERS: dict[Tier, TierConfig] = { + Tier.BRONZE: TierConfig( + required_credibility=0.70, + required_min_token_score=None, + required_unique_repos_count=3, + required_min_token_score_per_repo=5.0, # At least n initial unique repos must have at least x token score + credibility_scalar=1.0, + ), + Tier.SILVER: TierConfig( + required_credibility=0.65, + required_min_token_score=300.0, # Minimum total token score for Silver unlock + required_unique_repos_count=3, + required_min_token_score_per_repo=89.0, # At least n repos must have at least x token score + credibility_scalar=1.5, + ), + Tier.GOLD: TierConfig( + required_credibility=0.60, + required_min_token_score=500.0, # Minimum total token score for Gold unlock + required_unique_repos_count=3, + required_min_token_score_per_repo=144.0, # At least n unique repos must have at least x token score + credibility_scalar=2.0, + ), +} +TIERS_ORDER: list[Tier] = list(TIERS.keys()) + + +def get_next_tier(current: Tier) -> Optional[Tier]: + """Returns the next tier, or None if already at top.""" + idx = TIERS_ORDER.index(current) + if idx + 1 < len(TIERS_ORDER): + return TIERS_ORDER[idx + 1] + return None + + +def get_tier_from_config(tier_config: TierConfig) -> Optional[Tier]: + """Reverse lookup tier from TierConfig.""" + for tier, config in TIERS.items(): + if config == tier_config: + return tier + return None def allocate_emissions_by_tier(miner_evaluations: Dict[int, MinerEvaluation]) -> None: @@ -32,6 +143,9 @@ def allocate_emissions_by_tier(miner_evaluations: Dict[int, MinerEvaluation]) -> Args: miner_evaluations: Dict mapping uid to MinerEvaluation (modified in place) + + Note: MinerEvaluation is imported via TYPE_CHECKING for type hints only + (avoids circular import with gittensor.classes). """ # Step 1 & 2: Calculate net scores and network totals per tier network_tier_totals: Dict[Tier, float] = {tier: 0.0 for tier in TIERS_ORDER} diff --git a/gittensor/validator/storage/queries.py b/gittensor/validator/storage/queries.py index 777e087a..7a7ed34c 100644 --- a/gittensor/validator/storage/queries.py +++ b/gittensor/validator/storage/queries.py @@ -194,3 +194,36 @@ gold_leaf_score = EXCLUDED.gold_leaf_score, updated_at = NOW() """ + +# Merge Prediction Queries +UPSERT_MERGE_PREDICTION = """ +INSERT INTO merge_predictions ( + uid, hotkey, github_id, issue_id, repository, + pr_number, prediction, variance_at_prediction, timestamp +) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) +ON CONFLICT (uid, hotkey, github_id, issue_id, pr_number) +DO UPDATE SET + prediction = EXCLUDED.prediction, + variance_at_prediction = EXCLUDED.variance_at_prediction, + timestamp = EXCLUDED.timestamp +""" + +UPSERT_MERGE_PREDICTION_EMA = """ +INSERT INTO merge_prediction_emas (github_id, ema_score, rounds, updated_at) +VALUES (%s, %s, %s, %s) +ON CONFLICT (github_id) +DO UPDATE SET + ema_score = EXCLUDED.ema_score, + rounds = merge_prediction_emas.rounds + 1, + updated_at = EXCLUDED.updated_at +""" + +UPSERT_MERGE_SETTLED_ISSUE = """ +INSERT INTO merge_settled_issues (issue_id, outcome, merged_pr_number, settled_at) +VALUES (%s, %s, %s, %s) +ON CONFLICT (issue_id) DO NOTHING +""" + +DELETE_MERGE_PREDICTIONS_FOR_ISSUE = """ +DELETE FROM merge_predictions WHERE issue_id = %s +""" diff --git a/gittensor/validator/storage/repository.py b/gittensor/validator/storage/repository.py index da358047..999fff59 100644 --- a/gittensor/validator/storage/repository.py +++ b/gittensor/validator/storage/repository.py @@ -13,7 +13,7 @@ import numpy as np from gittensor.classes import FileChange, Issue, Miner, MinerEvaluation, PullRequest -from gittensor.validator.configurations.tier_config import Tier +from gittensor.validator.oss_contributions.tier_config import Tier from .queries import ( BULK_UPSERT_FILE_CHANGES, @@ -24,7 +24,11 @@ CLEANUP_STALE_MINER_EVALUATIONS, CLEANUP_STALE_MINER_TIER_STATS, CLEANUP_STALE_MINERS, + DELETE_MERGE_PREDICTIONS_FOR_ISSUE, SET_MINER, + UPSERT_MERGE_PREDICTION, + UPSERT_MERGE_PREDICTION_EMA, + UPSERT_MERGE_SETTLED_ISSUE, ) T = TypeVar('T') @@ -421,3 +425,46 @@ def set_miner_tier_stats(self, evaluation: MinerEvaluation) -> bool: self.db.rollback() self.logger.error(f'Error in miner tier stats storage: {e}') return False + + # Merge Prediction Storage + def store_merge_prediction( + self, + uid: int, + hotkey: str, + github_id: str, + issue_id: int, + repository: str, + pr_number: int, + prediction: float, + variance_at_prediction: float, + timestamp: str, + ) -> bool: + params = ( + uid, + hotkey, + github_id, + issue_id, + repository, + pr_number, + prediction, + variance_at_prediction, + timestamp, + ) + return self.set_entity(UPSERT_MERGE_PREDICTION, params) + + def store_merge_prediction_ema(self, github_id: str, ema_score: float, rounds: int, updated_at: str) -> bool: + params = (github_id, ema_score, rounds, updated_at) + return self.set_entity(UPSERT_MERGE_PREDICTION_EMA, params) + + def store_merge_settled_issue( + self, + issue_id: int, + outcome: str, + merged_pr_number: int | None, + settled_at: str, + ) -> bool: + params = (issue_id, outcome, merged_pr_number, settled_at) + return self.set_entity(UPSERT_MERGE_SETTLED_ISSUE, params) + + def delete_merge_predictions_for_issue(self, issue_id: int) -> bool: + return self.execute_command(DELETE_MERGE_PREDICTIONS_FOR_ISSUE, (issue_id,)) diff --git a/gittensor/validator/test/live_testnet/test_validator_live.py b/gittensor/validator/test/live_testnet/test_validator_live.py deleted file mode 100644 index c040d56a..00000000 --- a/gittensor/validator/test/live_testnet/test_validator_live.py +++ /dev/null @@ -1,211 +0,0 @@ -#!/usr/bin/env python3 -# The MIT License (MIT) -# Copyright © 2025 Entrius - -""" -Live Testnet Remote Debugging API - -This module provides a FastAPI endpoint to trigger validator scoring on-demand. -This allows you to: -1. Trigger scoring whenever you want (no waiting for 4-hour intervals) -2. Hit breakpoints in get_rewards() if debugpy is already attached -3. Optionally specify which UIDs to score - -IMPORTANT: This is FOR DEVELOPMENT ONLY. Enable via DEBUGPY_PORT in .env - -Security: - - All endpoints require API key authentication via X-API-Key header - - API key is auto-generated when you start validator with DEBUGPY_PORT set - - The key is displayed in the console and saved to .env - -Usage: - 1. Set DEBUGPY_PORT=5678 in your .env file - 2. Start your validator: bash scripts/run_validator.sh - 3. Confirm you want to enable debug mode (security prompt) - 4. Save the API key that is displayed in the console - 5. Attach your remote debugger to port 5678 (if you want breakpoints) - 6. Trigger scoring via API (include X-API-Key header): - - POST to http://remote_server:8099/trigger_scoring - - Optional: Include {"uids": [0, 1, 2]} in POST body to score specific UIDs - 7. Your breakpoints in get_rewards() will be hit - -Example: - # Trigger scoring for all UIDs - curl -X POST http://localhost:8099/trigger_scoring \\ - -H "X-API-Key: YOUR_API_KEY_HERE" - - # Trigger scoring for specific UIDs - curl -X POST http://localhost:8099/trigger_scoring \\ - -H "X-API-Key: YOUR_API_KEY_HERE" \\ - -H "Content-Type: application/json" \\ - -d '{"uids": [0, 5, 10]}' - - # Health check - curl http://localhost:8099/health \\ - -H "X-API-Key: YOUR_API_KEY_HERE" -""" - -import os -from typing import TYPE_CHECKING, Any, Dict, List, Optional - -import bittensor as bt -import numpy as np -import uvicorn -from fastapi import Body, Depends, FastAPI, Header, HTTPException - -from gittensor.validator.utils.load_weights import load_master_repo_weights, load_programming_language_weights - -if TYPE_CHECKING: - from neurons.base.validator import BaseValidatorNeuron - -from gittensor.utils.uids import get_all_uids -from gittensor.validator.evaluation.reward import get_rewards - - -def create_debug_api(validator: 'BaseValidatorNeuron', port: int = 8099): - """ - Create a FastAPI application for triggering validator scoring on-demand. - - Args: - validator: The validator instance to use for scoring - port: Port to run the FastAPI server on (default: 8099) - - Returns: - FastAPI app instance - """ - app = FastAPI(title='Gittensor Validator Debug API') - - # Load API key from environment - REQUIRED_API_KEY = os.getenv('VALIDATOR_DEBUG_API_KEY') - - def verify_api_key(x_api_key: Optional[str] = Header(None)): - """Verify the API key provided in the X-API-Key header.""" - if not REQUIRED_API_KEY: - bt.logging.error('VALIDATOR_DEBUG_API_KEY not set in environment!') - raise HTTPException(status_code=500, detail='API key not configured on server') - - if not x_api_key: - bt.logging.warning('API request rejected: No API key provided') - raise HTTPException(status_code=401, detail='Missing API key. Provide X-API-Key header.') - - if x_api_key != REQUIRED_API_KEY: - bt.logging.warning('API request rejected: Invalid API key') - raise HTTPException(status_code=403, detail='Invalid API key') - - return True - - @app.get('/health') - async def health(authorized: bool = Depends(verify_api_key)): - """Health check endpoint (requires API key).""" - return { - 'status': 'healthy', - 'validator_uid': int(validator.uid) if validator.uid is not None else None, - 'network': ( - str(validator.config.subtensor.chain_endpoint) if hasattr(validator.config, 'subtensor') else 'unknown' - ), - 'netuid': int(validator.config.netuid) if hasattr(validator.config, 'netuid') else None, - } - - @app.post('/trigger_scoring') - async def trigger_scoring( - uids: Optional[List[int]] = Body(None, description='Optional list of UIDs to score'), - authorized: bool = Depends(verify_api_key), - ): - """ - Manually trigger a scoring round. If debugpy is already attached, breakpoints will be hit. - - Args: - uids: Optional list of specific UIDs to score - - Returns: - JSON with scoring results - """ - try: - # Check if running on testnet - chain_endpoint = validator.config.subtensor.chain_endpoint - # TODO: Make this a sufficient is testnet check, below one doesn't seem to be working. - # is_testnet = "test" in chain_endpoint.lower() or chain_endpoint == "wss://test.finney.opentensor.ai:443/" - is_testnet = True - - if not is_testnet: - bt.logging.error(f'Remote debugging endpoint blocked: Not running on testnet (chain: {chain_endpoint})') - return { - 'error': 'Not allowed on mainnet', - 'message': 'This endpoint is only available when validator is running on testnet', - 'current_chain': str(chain_endpoint), - } - - bt.logging.info(f'Testnet check passed: {chain_endpoint}') - - # get the master repo list - master_repositories: Dict[str, Dict[str, Any]] = load_master_repo_weights() - programming_languages = load_programming_language_weights() - - # Get UIDs to score - if uids is not None: - miner_uids = list(uids) - bt.logging.info(f'Scoring specific UIDs: {miner_uids}') - else: - miner_uids = get_all_uids(validator) - bt.logging.info(f'Scoring all UIDs: {len(miner_uids)} miners') - - # Check if debugpy is attached - try: - import debugpy - - if debugpy.is_client_connected(): - bt.logging.info('Debugger is attached - breakpoints will be hit!') - else: - bt.logging.info('No debugger attached - running normally') - except ImportError: - bt.logging.info('debugpy not installed - running normally') - - # Trigger scoring - THIS IS WHERE YOUR BREAKPOINTS WILL HIT - bt.logging.info('***** Starting manual scoring round *****') - rewards = await get_rewards(validator, miner_uids, master_repositories, programming_languages) - - # Format results - ensure all values are JSON serializable - result = { - 'status': 'success', - 'uids_scored': [int(uid) for uid in miner_uids], - 'total_uids': len(miner_uids), - 'total_reward_sum': float(np.sum(rewards)) if len(rewards) > 0 else 0.0, - 'non_zero_rewards': int(np.count_nonzero(rewards)) if len(rewards) > 0 else 0, - 'rewards': {str(int(uid)): float(reward) for uid, reward in zip(miner_uids, rewards)}, - } - - bt.logging.info(f'Scoring complete! Total rewards: {result["total_reward_sum"]:.2f}') - return result - - except Exception as e: - bt.logging.error(f'Scoring failed: {e}') - import traceback - - return {'error': str(e), 'traceback': traceback.format_exc()} - - return app - - -def start_debug_api(validator: 'BaseValidatorNeuron', port: int = 8099): - """ - Start the debug API server for on-demand scoring. - - This should be called from the validator's __init__ when DEBUGPY_PORT is set in .env. - - Args: - validator: The validator instance - port: Port to run FastAPI server on - """ - app = create_debug_api(validator, port) - - bt.logging.info('=' * 70) - bt.logging.info('REMOTE DEBUGGING API ENABLED (FOR DEVELOPMENT ONLY)') - bt.logging.info('=' * 70) - bt.logging.info(f'API endpoint: http://0.0.0.0:{port}') - bt.logging.info(f'Health check: curl http://localhost:{port}/health') - bt.logging.info(f'Trigger scoring: curl -X POST http://localhost:{port}/trigger_scoring') - bt.logging.info(f'API docs: http://localhost:{port}/docs') - bt.logging.info('=' * 70) - - # Run FastAPI app with uvicorn (blocking) - uvicorn.run(app, host='0.0.0.0', port=port, log_level='info') diff --git a/gittensor/validator/test/simulation/__init__.py b/gittensor/validator/test/simulation/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/gittensor/validator/test/simulation/mock_evaluations.py b/gittensor/validator/test/simulation/mock_evaluations.py deleted file mode 100644 index fb1f53a5..00000000 --- a/gittensor/validator/test/simulation/mock_evaluations.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 -# The MIT License (MIT) -# Copyright © 2025 Entrius - -""" -Custom Mock MinerEvaluation Objects for Simulation Testing - -Provides a template for custom MinerEvaluation objects to test alongside DB data. -Uncomment and modify the EXAMPLE_MINER template below. -""" - -import os -import sys -from typing import Dict - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) - -from gittensor.classes import MinerEvaluation - - -def get_custom_evaluations() -> Dict[int, MinerEvaluation]: - """ - Return custom MinerEvaluation objects for testing. - """ - custom_evaluations = {} - - # ========================================================================= - # EXAMPLE TEMPLATE - Uncomment and modify as needed - # Shows merged, open, and closed PRs with file changes and issues - # ========================================================================= - - # EXAMPLE_MINER = MinerEvaluation(uid=9001, hotkey="5CustomHotkey...", github_id="custom_user") - # for tier in TIERS.keys(): - # EXAMPLE_MINER.stats_by_tier[tier] = TierStats() - # - # # --- MERGED PR (earns score) --- - # merged_pr = PullRequest( - # number=1001, - # repository_full_name="opentensor/bittensor", # Must exist in master_repositories.json - # uid=9001, - # hotkey="5CustomHotkey...", - # github_id="custom_user", - # title="Add feature", - # author_login="custom_user", - # merged_at=datetime.now(timezone.utc) - timedelta(days=5), - # created_at=datetime.now(timezone.utc) - timedelta(days=7), - # pr_state=PRState.MERGED, - # additions=150, - # deletions=30, - # commits=3, - # issues=[ - # Issue( - # number=100, - # pr_number=1001, - # repository_full_name="opentensor/bittensor", - # title="Bug report", - # created_at=datetime.now(timezone.utc) - timedelta(days=60), - # closed_at=datetime.now(timezone.utc) - timedelta(days=5), - # author_login="other_user", # Must differ from PR author - # state="CLOSED", - # ), - # ], - # ) - # merged_pr.set_file_changes([ - # FileChange(pr_number=1001, repository_full_name="opentensor/bittensor", - # filename="src/feature.py", changes=100, additions=80, deletions=20, status="modified"), - # FileChange(pr_number=1001, repository_full_name="opentensor/bittensor", - # filename="tests/test_feature.py", changes=80, additions=70, deletions=10, status="added"), - # ]) - # EXAMPLE_MINER.merged_pull_requests.append(merged_pr) - # EXAMPLE_MINER.unique_repos_contributed_to.add(merged_pr.repository_full_name) - # - # # --- OPEN PR (collateral deduction) --- - # open_pr = PullRequest( - # number=1002, - # repository_full_name="opentensor/bittensor", - # uid=9001, - # hotkey="5CustomHotkey...", - # github_id="custom_user", - # title="WIP feature", - # author_login="custom_user", - # merged_at=None, - # created_at=datetime.now(timezone.utc) - timedelta(days=3), - # pr_state=PRState.OPEN, - # additions=50, - # deletions=10, - # ) - # open_pr.set_file_changes([ - # FileChange(pr_number=1002, repository_full_name="opentensor/bittensor", - # filename="src/new.py", changes=60, additions=50, deletions=10, status="added"), - # ]) - # EXAMPLE_MINER.open_pull_requests.append(open_pr) - # - # # --- CLOSED PR (affects credibility) --- - # closed_pr = PullRequest( - # number=1003, - # repository_full_name="opentensor/bittensor", - # uid=9001, - # hotkey="5CustomHotkey...", - # github_id="custom_user", - # title="Rejected PR", - # author_login="custom_user", - # merged_at=None, - # created_at=datetime.now(timezone.utc) - timedelta(days=10), - # pr_state=PRState.CLOSED, - # additions=20, - # deletions=5, - # ) - # closed_pr.set_file_changes([ - # FileChange(pr_number=1003, repository_full_name="opentensor/bittensor", - # filename="rejected.py", changes=25, additions=20, deletions=5, status="added"), - # ]) - # EXAMPLE_MINER.closed_pull_requests.append(closed_pr) - # - # custom_evaluations[9001] = EXAMPLE_MINER - - return custom_evaluations diff --git a/gittensor/validator/test/simulation/run_scoring_simulation.py b/gittensor/validator/test/simulation/run_scoring_simulation.py deleted file mode 100644 index cb25757d..00000000 --- a/gittensor/validator/test/simulation/run_scoring_simulation.py +++ /dev/null @@ -1,345 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2025 Entrius - -""" -Scoring Simulation Runner - -Runs the full incentive mechanism scoring pipeline: -1. Loads miner identities (uid, hotkey, github_id) from the test database -2. Fetches PRs from GitHub in real-time using the production load_miners_prs function -3. Scores PRs using token-based AST scoring with file contents from GitHub - -This tests the complete production flow including GitHub API integration. - -Usage: - python run_scoring_simulation.py - -Configuration: - - DB connection via env vars: DB_HOST, DB_PORT, DB_USER, DB_PASSWORD, DB_NAME - - GitHub PAT via GITHUB_PAT or GITHUB_TOKEN env var (required) - - Custom evaluations in mock_evaluations.py -""" - -import os -import sys -import time -from datetime import datetime, timezone -from pathlib import Path -from typing import Dict, List, Tuple - -import bittensor as bt -from dotenv import load_dotenv - -from gittensor.classes import MinerEvaluation -from gittensor.utils.github_api_tools import load_miners_prs -from gittensor.validator.configurations.tier_config import TIERS, TierStats -from gittensor.validator.evaluation.dynamic_emissions import apply_dynamic_emissions_using_network_contributions -from gittensor.validator.evaluation.inspections import detect_and_penalize_miners_sharing_github -from gittensor.validator.evaluation.normalize import normalize_rewards_linear -from gittensor.validator.evaluation.scoring import finalize_miner_scores, score_miner_prs -from gittensor.validator.utils.load_weights import ( - load_master_repo_weights, - load_programming_language_weights, - load_token_config, -) -from gittensor.validator.utils.storage import DatabaseStorage - - -def make_aware(dt: datetime) -> datetime: - """Convert naive datetime to UTC-aware. Returns None if input is None.""" - if dt is None: - return None - if dt.tzinfo is None: - return dt.replace(tzinfo=timezone.utc) - return dt - - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) - - -validator_env = Path(__file__).parent.parent.parent / '.env' -load_dotenv(validator_env) - - -# Enable bittensor logging to console -bt.logging.set_debug(True) - -# Set to an integer to limit the number of miners to score (e.g., 2, 5, 10) -# Set to None to score all miners -MINER_LIMIT = None - -# Set to empty list [] to score all miners (subject to MINER_LIMIT) -SPECIFIC_GITHUB_IDS = [] - - -try: - from gittensor.validator.test.simulation.mock_evaluations import get_custom_evaluations - - CUSTOM_EVALUATIONS_AVAILABLE = True -except ImportError: - CUSTOM_EVALUATIONS_AVAILABLE = False - - -# ============================================================================= -# Database Queries -# ============================================================================= - -LOAD_ALL_MINERS = """ -SELECT DISTINCT m.uid, m.hotkey, m.github_id, me.total_score -FROM miners m -INNER JOIN pull_requests pr ON m.uid = pr.uid AND m.hotkey = pr.hotkey AND m.github_id = pr.github_id -INNER JOIN miner_evaluations me ON m.uid = me.uid AND m.hotkey = me.hotkey AND m.github_id = me.github_id -""" - - -def create_db_connection(): - """Create database connection from env vars.""" - try: - import psycopg2 - except ImportError: - print('ERROR: psycopg2 not installed') - return None - - try: - conn = psycopg2.connect( - host=os.getenv('DB_HOST', 'localhost'), - port=int(os.getenv('DB_PORT', 5432)), - user=os.getenv('DB_USER', 'postgres'), - password=os.getenv('DB_PASSWORD', ''), - database=os.getenv('DB_NAME', 'gittensor_validator'), - ) - conn.autocommit = False - print(f'Connected to {os.getenv("DB_NAME", "gittensor_validator")}') - return conn - except Exception as e: - print(f'ERROR: DB connection failed: {e}') - return None - - -def load_miner_identities(conn) -> List[Tuple[int, str, str]]: - """Load miner identities (uid, hotkey, github_id) from database. - - Applies filters based on SPECIFIC_GITHUB_IDS and MINER_LIMIT configuration. - If SPECIFIC_GITHUB_IDS are provided, bypasses DB entirely and creates mock identities - to fetch PRs directly from GitHub GraphQL. - """ - # If specific github_ids are provided, bypass DB and create mock identities directly - # This allows fetching fresh PRs from GraphQL without requiring DB records - if SPECIFIC_GITHUB_IDS: - miners = [] - print(f' Using SPECIFIC_GITHUB_IDS (bypassing DB): {SPECIFIC_GITHUB_IDS}') - for i, github_id in enumerate(SPECIFIC_GITHUB_IDS): - mock_uid = -(i + 1) - mock_hotkey = f'mock_hotkey_{github_id}' - # Include dummy total_score (0.0) to match DB query result format - miners.append((mock_uid, mock_hotkey, github_id, 0.0)) - print(f' -> Mock miner: uid={mock_uid}, github_id={github_id}') - - # Apply miner limit if configured - if MINER_LIMIT is not None and len(miners) > MINER_LIMIT: - miners = miners[:MINER_LIMIT] - print(f' Limited to {len(miners)} miners by MINER_LIMIT') - - return miners - - # Otherwise, load from database - cur = conn.cursor() - cur.execute(LOAD_ALL_MINERS) - miners = cur.fetchall() - cur.close() - total_in_db = len(miners) - print(f' Found {total_in_db} miners with PRs in DB') - - # Apply miner limit if configured - if MINER_LIMIT is not None and len(miners) > MINER_LIMIT: - miners = miners[:MINER_LIMIT] - print(f' Limited to {len(miners)} miners by MINER_LIMIT') - - return miners - - -def create_miner_evaluation(uid: int, hotkey: str, github_id: str, github_pat: str) -> MinerEvaluation: - """Create a MinerEvaluation with identity info and GitHub PAT.""" - # Ensure github_id is a string (may be int from config or DB) - github_id_str = str(github_id) if github_id else '0' - miner_eval = MinerEvaluation(uid=uid, hotkey=hotkey, github_id=github_id_str, github_pat=github_pat) - for tier in TIERS.keys(): - miner_eval.stats_by_tier[tier] = TierStats() - return miner_eval - - -# ============================================================================= -# Main Simulation -# ============================================================================= - - -def run_scoring_simulation( - include_custom: bool = True, - store_evaluations: bool = False, -) -> Tuple[Dict[int, MinerEvaluation], Dict[int, float], Dict[int, float]]: - """Run full scoring pipeline on DB data. - - Args: - include_custom: Include custom evaluations from mock_evaluations.py - store_evaluations: If True, store evaluations to database via DatabaseStorage - """ - print('=' * 70) - print('SCORING SIMULATION START') - print('=' * 70) - sys.stdout.flush() - time.sleep(0.1) - - # 1. Load weights and GitHub PAT - print('\n[1/8] Loading weights and configuration...') - sys.stdout.flush() - time.sleep(0.1) - master_repos = load_master_repo_weights() - prog_langs = load_programming_language_weights() - token_config = load_token_config() - github_pat = os.getenv('GITHUB_PAT') or os.getenv('GITHUB_TOKEN') - print(f' {len(master_repos)} repos, {len(prog_langs)} languages') - print( - f' Token config: {len(token_config.structural_bonus)} structural, {len(token_config.leaf_tokens)} leaf types' - ) - if not github_pat: - print(' ERROR: No GitHub PAT set - cannot fetch PRs from GitHub') - return {}, {}, {} - print(' GitHub PAT: Available') - sys.stdout.flush() - time.sleep(0.1) - - # 2. Connect to DB to get miner identities - print('\n[2/8] Loading miner identities from DB...') - sys.stdout.flush() - time.sleep(0.1) - conn = create_db_connection() - if not conn: - return {}, {}, {} - miners = load_miner_identities(conn) - conn.close() - - # 3. Create evaluations and fetch PRs from GitHub - print('\n[3/8] Fetching PRs from GitHub for each miner...') - sys.stdout.flush() - time.sleep(0.1) - evals: Dict[int, MinerEvaluation] = {} - for uid, hotkey, github_id, _ in miners: - print(f' Fetching PRs for uid={uid} ({github_id})...') - ev = create_miner_evaluation(uid, hotkey, github_id, github_pat) - load_miners_prs(ev, master_repos, max_prs=100) - evals[uid] = ev - print(f' -> {ev.total_merged_prs} merged, {ev.total_open_prs} open, {ev.total_closed_prs} closed') - - # 4. Add custom evaluations - if include_custom and CUSTOM_EVALUATIONS_AVAILABLE: - print('\n[4/8] Adding custom evaluations...') - for uid, ev in get_custom_evaluations().items(): - ev.github_pat = github_pat - evals[uid] = ev - print(f' Added custom uid={uid}') - else: - print('\n[4/9] No custom evaluations.') - sys.stdout.flush() - time.sleep(0.1) - - # 5. Score PRs (uses token-based scoring with file contents from GitHub) - print('\n[5/8] Scoring PRs with token-based scoring...') - sys.stdout.flush() - time.sleep(0.1) - for uid, ev in evals.items(): - if ev.failed_reason: - continue - score_miner_prs(ev, master_repos, prog_langs, token_config) - - # 6. Detect duplicate GitHub accounts - print('\n[6/9] Checking for duplicate GitHub accounts...') - sys.stdout.flush() - time.sleep(0.1) - detect_and_penalize_miners_sharing_github(evals) - - # 7. Finalize scores - print('\n[7/9] Finalizing scores...') - sys.stdout.flush() - time.sleep(0.1) - finalize_miner_scores(evals) - - # 8. Normalize & apply dynamic emissions - print('\n[8/9] Normalizing & applying dynamic emissions...') - sys.stdout.flush() - time.sleep(0.1) - normalized = normalize_rewards_linear(evals) - scaled = apply_dynamic_emissions_using_network_contributions(normalized, evals) - - # 9. Store evaluations (optional) - if store_evaluations: - print('\n[9/9] Storing evaluations to database...') - sys.stdout.flush() - time.sleep(0.1) - db_storage = DatabaseStorage() - if db_storage.is_enabled(): - for uid, miner_eval in evals.items(): - result = db_storage.store_evaluation(miner_eval) - if result.success: - print(f' Stored UID {uid}') - else: - print(f' Failed UID {uid}: {result.errors}') - db_storage.close() - else: - print(' WARNING: Database storage not enabled. Check DB env vars.') - else: - print('\n[9/9] Skipping evaluation storage (store_evaluations=False)') - - # Print summary - _print_summary(evals, normalized, scaled) - - print('\n' + '=' * 70) - print('SCORING SIMULATION COMPLETE') - print('=' * 70) - - return evals, normalized, scaled - - -def _print_summary(evals: Dict[int, MinerEvaluation], normalized: Dict[int, float], scaled: Dict[int, float]): - """Print results summary.""" - print('\n' + '=' * 70) - print('RESULTS SUMMARY') - print('=' * 70) - - sorted_uids = sorted(scaled.keys(), key=lambda u: scaled.get(u, 0), reverse=True) - - print(f'\n{"UID":<6} {"GitHub":<18} {"Tier":<7} {"Merged":<7} {"Score":<10} {"Normalized":<12} {"Scaled":<10}') - print('-' * 82) - - for uid in sorted_uids: - ev = evals.get(uid) - if not ev: - continue - tier = ev.current_tier.value if ev.current_tier else 'None' - print( - f'{uid:<6} {(ev.github_id or "N/A"):<18} {tier:<7} {ev.total_merged_prs:<7} ' - f'{ev.total_score:<10.2f} {normalized.get(uid, 0):<12.6f} {scaled.get(uid, 0):<10.6f}' - ) - - -if __name__ == '__main__': - import argparse - - parser = argparse.ArgumentParser(description='Run scoring simulation') - parser.add_argument('--store', action='store_true', help='Store evaluations to database') - parser.add_argument('--no-custom', action='store_true', help='Exclude custom evaluations') - args = parser.parse_args() - - try: - run_scoring_simulation( - include_custom=not args.no_custom, - store_evaluations=args.store, - ) - except KeyboardInterrupt: - print('\nInterrupted') - sys.exit(0) - except Exception as e: - print(f'ERROR: {e}') - import traceback - - traceback.print_exc() - sys.exit(1) diff --git a/gittensor/validator/utils/config.py b/gittensor/validator/utils/config.py index 39d11f75..e0dde779 100644 --- a/gittensor/validator/utils/config.py +++ b/gittensor/validator/utils/config.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import bittensor as bt @@ -14,6 +15,9 @@ # optional env vars STORE_DB_RESULTS = os.getenv('STORE_DB_RESULTS', 'false').lower() == 'true' +# Merge predictions DB path — defaults to /app/data/ so it lands inside the Docker volume +MP_DB_PATH = os.getenv('MP_DB_PATH', str(Path(__file__).resolve().parents[3] / 'data' / 'gt-merge-preds.db')) + # log values bt.logging.info(f'VALIDATOR_WAIT: {VALIDATOR_WAIT}') bt.logging.info(f'VALIDATOR_STEPS_INTERVAL: {VALIDATOR_STEPS_INTERVAL}') diff --git a/gittensor/validator/utils/github_validation.py b/gittensor/validator/utils/github_validation.py new file mode 100644 index 00000000..5fbda02b --- /dev/null +++ b/gittensor/validator/utils/github_validation.py @@ -0,0 +1,30 @@ +# The MIT License (MIT) +# Copyright © 2025 Entrius + +"""Shared GitHub credential validation used by multiple validator subsystems.""" + +from typing import Optional, Tuple + +from gittensor.constants import MIN_GITHUB_ACCOUNT_AGE +from gittensor.utils.github_api_tools import ( + get_github_account_age_days, + get_github_id, +) + + +def validate_github_credentials(uid: int, pat: Optional[str]) -> Tuple[Optional[str], Optional[str]]: + """Validate PAT and return (github_id, error_reason) tuple.""" + if not pat: + return None, f'No Github PAT provided by miner {uid}' + + github_id = get_github_id(pat) + if not github_id: + return None, f"No Github id found for miner {uid}'s PAT" + + account_age = get_github_account_age_days(pat) + if not account_age: + return None, f'Could not determine Github account age for miner {uid}' + if account_age < MIN_GITHUB_ACCOUNT_AGE: + return None, f"Miner {uid}'s Github account too young ({account_age} < {MIN_GITHUB_ACCOUNT_AGE} days)" + + return github_id, None diff --git a/gittensor/validator/utils/load_weights.py b/gittensor/validator/utils/load_weights.py index b02f6ec1..e9763c9c 100644 --- a/gittensor/validator/utils/load_weights.py +++ b/gittensor/validator/utils/load_weights.py @@ -8,7 +8,7 @@ import bittensor as bt from gittensor.constants import NON_CODE_EXTENSIONS -from gittensor.validator.configurations.tier_config import Tier +from gittensor.validator.oss_contributions.tier_config import Tier @dataclass diff --git a/gittensor/validator/utils/storage.py b/gittensor/validator/utils/storage.py index f08c94e2..1f9dd057 100644 --- a/gittensor/validator/utils/storage.py +++ b/gittensor/validator/utils/storage.py @@ -84,6 +84,69 @@ def store_evaluation(self, miner_eval: MinerEvaluation) -> StorageResult: return result + def store_merge_prediction( + self, + uid: int, + hotkey: str, + github_id: str, + issue_id: int, + repository: str, + pr_number: int, + prediction: float, + variance_at_prediction: float, + timestamp: str, + ) -> bool: + if not self.is_enabled(): + return False + try: + return self.repo.store_merge_prediction( + uid, + hotkey, + github_id, + issue_id, + repository, + pr_number, + prediction, + variance_at_prediction, + timestamp, + ) + except Exception as e: + self.logger.warning(f'Postgres merge prediction write failed (non-fatal): {e}') + return False + + def store_merge_prediction_ema(self, github_id: str, ema_score: float, rounds: int, updated_at: str) -> bool: + if not self.is_enabled(): + return False + try: + return self.repo.store_merge_prediction_ema(github_id, ema_score, rounds, updated_at) + except Exception as e: + self.logger.warning(f'Postgres merge prediction EMA write failed (non-fatal): {e}') + return False + + def store_merge_settled_issue( + self, + issue_id: int, + outcome: str, + merged_pr_number: int | None, + settled_at: str, + ) -> bool: + if not self.is_enabled(): + return False + try: + return self.repo.store_merge_settled_issue(issue_id, outcome, merged_pr_number, settled_at) + except Exception as e: + self.logger.warning(f'Postgres merge settled issue write failed (non-fatal): {e}') + return False + + def delete_merge_predictions_for_issue(self, issue_id: int) -> bool: + if not self.is_enabled(): + return False + try: + return self.repo.delete_merge_predictions_for_issue(issue_id) + except Exception as e: + self.logger.warning(f'Postgres merge prediction delete failed (non-fatal): {e}') + return False + def _log_storage_summary(self, counts: Dict[str, int]): """Log a summary of what was stored""" self.logger.info('Storage Summary:') diff --git a/gittensor/validator/weights/programming_languages.json b/gittensor/validator/weights/programming_languages.json index bfec7882..b4ed3a9b 100644 --- a/gittensor/validator/weights/programming_languages.json +++ b/gittensor/validator/weights/programming_languages.json @@ -61,10 +61,10 @@ "ipynb": { "weight": 0.3 }, "java": { "weight": 1.75, "language": "java" }, "jl": { "weight": 1.0, "language": "julia" }, - "js": { "weight": 1.3, "language": "javascript" }, + "js": { "weight": 1.05, "language": "javascript" }, "json": { "weight": 0.1 }, "jsonc": { "weight": 0.1 }, - "jsx": { "weight": 1.3, "language": "javascript" }, + "jsx": { "weight": 1.1, "language": "javascript" }, "kt": { "weight": 1.75, "language": "kotlin" }, "kts": { "weight": 1.75, "language": "kotlin" }, "less": { "weight": 0.95, "language": "css" }, @@ -81,7 +81,7 @@ "mli": { "weight": 1.75, "language": "ocaml" }, "mm": { "weight": 1.75, "language": "objc" }, "move": { "weight": 1.0 }, - "mts": { "weight": 1.5, "language": "typescript" }, + "mts": { "weight": 1.05, "language": "typescript" }, "nim": { "weight": 1.5 }, "nix": { "weight": 1.0, "language": "nix" }, "pas": { "weight": 1.5, "language": "pascal" }, @@ -119,9 +119,9 @@ "text": { "weight": 0.08 }, "tf": { "weight": 1.0, "language": "hcl" }, "toml": { "weight": 0.5 }, - "ts": { "weight": 1.5, "language": "typescript" }, + "ts": { "weight": 1.05, "language": "typescript" }, "tsv": { "weight": 0.1 }, - "tsx": { "weight": 1.5, "language": "tsx" }, + "tsx": { "weight": 1.1, "language": "tsx" }, "txt": { "weight": 0.08 }, "v": { "weight": 1.5, "language": "v" }, "vhd": { "weight": 1.75, "language": "vhdl" }, diff --git a/neurons/base/validator.py b/neurons/base/validator.py index d7faba0e..3c1484d6 100644 --- a/neurons/base/validator.py +++ b/neurons/base/validator.py @@ -95,7 +95,7 @@ def serve_axon(self): axon=self.axon, ) bt.logging.info( - f'Running validator {self.axon} on network: {self.config.subtensor.chain_endpoint} with netuid: {self.config.netuid}' + f'Axon served on network: {self.config.subtensor.chain_endpoint} with netuid: {self.config.netuid}' ) except Exception as e: bt.logging.error(f'Failed to serve Axon with exception: {e}') @@ -132,6 +132,15 @@ def run(self): # Check that validator is registered on the network. self.sync() + # Start the axon after full initialization (subclass __init__ complete). + if not self.config.neuron.axon_off and hasattr(self, 'axon'): + self.axon.start() + bt.logging.info( + f'Running validator {self.axon} on network: {self.config.subtensor.chain_endpoint} with netuid: {self.config.netuid}' + ) + else: + bt.logging.info('Validator axon not starting, continuing with validator process anyways.') + bt.logging.info(f'Validator starting at block: {self.block}') # This loop maintains the validator's operations until intentionally stopped. diff --git a/neurons/miner.py b/neurons/miner.py index c9d36b32..42c82573 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -98,4 +98,4 @@ async def priority(self, synapse: GitPatSynapse) -> float: while True: bt.logging.info('Gittensor miner running...') - time.sleep(45) + time.sleep(100) diff --git a/neurons/validator.py b/neurons/validator.py index 8bef671b..284664d4 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -16,8 +16,8 @@ # DEALINGS IN THE SOFTWARE. -import threading import time +from functools import partial from typing import Dict, List, Set import bittensor as bt @@ -26,6 +26,8 @@ from gittensor.__init__ import __version__ from gittensor.classes import MinerEvaluation, MinerEvaluationCache from gittensor.validator.forward import forward +from gittensor.validator.merge_predictions.handler import blacklist_prediction, handle_prediction, priority_prediction +from gittensor.validator.merge_predictions.mp_storage import PredictionStorage from gittensor.validator.utils.config import STORE_DB_RESULTS, WANDB_PROJECT, WANDB_VALIDATOR_NAME from gittensor.validator.utils.storage import DatabaseStorage from neurons.base.validator import BaseValidatorNeuron @@ -44,6 +46,18 @@ class Validator(BaseValidatorNeuron): def __init__(self, config=None): super(Validator, self).__init__(config=config) + # Merge predictions — SQLite storage + axon handler + self.mp_storage = PredictionStorage() + if hasattr(self, 'axon') and self.axon is not None: + self.axon.attach( + forward_fn=partial(handle_prediction, self), + blacklist_fn=partial(blacklist_prediction, self), + priority_fn=partial(priority_prediction, self), + ) + bt.logging.info('Merge predictions handler attached to axon') + else: + bt.logging.warning('Axon not available, skipping prediction handler attachment') + # Init in-memory cache for miner evaluations (fallback when GitHub API fails) self.evaluation_cache = MinerEvaluationCache() @@ -53,21 +67,6 @@ def __init__(self, config=None): bt.logging.warning('Validation result storage enabled.') self.db_storage = DatabaseStorage() - # Init remote debugging API (FOR DEVELOPMENT ONLY). Requires DEBUGPY_PORT in .env - if self.config.neuron.remote_debug_port is not None: - from gittensor.validator.test.live_testnet.test_validator_live import start_debug_api - - bt.logging.warning('Remote debugging api enabled.') - - # Start debug API in background thread - debug_thread = threading.Thread( - target=start_debug_api, - args=(self, self.config.neuron.remote_debug_port), - daemon=True, - name='RemoteDebugAPI', - ) - debug_thread.start() - # Initialize wandb only if disable_set_weights is False if not self.config.neuron.disable_set_weights: try: diff --git a/scripts/vali-entrypoint.sh b/scripts/vali-entrypoint.sh index fd378b49..15f540f7 100755 --- a/scripts/vali-entrypoint.sh +++ b/scripts/vali-entrypoint.sh @@ -6,7 +6,7 @@ if [ -z "$HOTKEY_NAME" ]; then echo "HOTKEY_NAME is not set" && exit 1; fi if [ -z "$SUBTENSOR_NETWORK" ]; then echo "SUBTENSOR_NETWORK is not set" && exit 1; fi if [ -z "$PORT" ]; then echo "PORT is not set" && exit 1; fi if [ -z "$LOG_LEVEL" ]; then echo "LOG_LEVEL is not set" && exit 1; fi -# if [ -z "$GITTENSOR_VALIDATOR_PAT" ]; then echo "GITTENSOR_VALIDATOR_PAT is not set" && exit 1; fi +if [ -z "$GITTENSOR_VALIDATOR_PAT" ]; then echo "GITTENSOR_VALIDATOR_PAT is not set" && exit 1; fi exec python neurons/validator.py \ --netuid ${NETUID} \ diff --git a/tests/cli/test_issue_predict.py b/tests/cli/test_issue_predict.py index f180b446..03c6e429 100644 --- a/tests/cli/test_issue_predict.py +++ b/tests/cli/test_issue_predict.py @@ -11,6 +11,7 @@ def test_predict_interactive_continue_cancel_skips_miner_validation(cli_root, ru with ( patch('gittensor.cli.issue_commands.predict.get_contract_address', return_value='0xabc'), patch('gittensor.cli.issue_commands.predict.resolve_network', return_value=('ws://x', 'test')), + patch('gittensor.cli.issue_commands.predict.resolve_netuid_from_contract', return_value=1), patch('gittensor.cli.issue_commands.predict.fetch_issue_from_contract', return_value=sample_issue), patch('gittensor.cli.issue_commands.predict.fetch_open_issue_pull_requests', return_value=sample_prs), patch('gittensor.cli.issue_commands.predict._is_interactive', return_value=True), @@ -32,15 +33,20 @@ def test_predict_json_success_payload_schema(cli_root, runner, sample_issue, sam with ( patch('gittensor.cli.issue_commands.predict.get_contract_address', return_value='0xabc'), patch('gittensor.cli.issue_commands.predict.resolve_network', return_value=('ws://x', 'test')), + patch('gittensor.cli.issue_commands.predict.resolve_netuid_from_contract', return_value=1), patch('gittensor.cli.issue_commands.predict.fetch_issue_from_contract', return_value=sample_issue), patch('gittensor.cli.issue_commands.predict.fetch_open_issue_pull_requests', return_value=sample_prs), patch( 'gittensor.cli.issue_commands.predict._resolve_registered_miner_hotkey', return_value='5FakeHotkey123', ), - patch('gittensor.cli.issue_commands.predict.broadcast_predictions_stub') as mock_broadcast_stub, + patch('gittensor.cli.issue_commands.predict.broadcast_predictions') as mock_broadcast_stub, ): - mock_broadcast_stub.side_effect = lambda payload: payload + mock_broadcast_stub.return_value = { + 'issue_id': 42, + 'repository': 'entrius/gittensor', + 'predictions': {'101': 0.7}, + } result = runner.invoke( cli_root, ['issues', 'predict', '--id', '42', '--pr', '101', '--probability', '0.7', '--json'], @@ -49,11 +55,12 @@ def test_predict_json_success_payload_schema(cli_root, runner, sample_issue, sam assert result.exit_code == 0 mock_broadcast_stub.assert_called_once() - payload = json.loads(result.output) - assert set(payload.keys()) == {'issue_id', 'repository', 'predictions'} + call_kwargs = mock_broadcast_stub.call_args + payload = call_kwargs.kwargs['payload'] + assert {'issue_id', 'repository', 'predictions'} <= set(payload.keys()) assert payload['issue_id'] == 42 assert payload['repository'] == 'entrius/gittensor' - assert {int(k): v for k, v in payload['predictions'].items()} == {101: 0.7} + assert payload['predictions'] == {101: 0.7} def test_predict_json_requires_non_interactive_inputs(runner, cli_root): @@ -108,6 +115,7 @@ def test_predict_rejects_pr_not_in_open_set_before_miner_validation(cli_root, ru with ( patch('gittensor.cli.issue_commands.predict.get_contract_address', return_value='0xabc'), patch('gittensor.cli.issue_commands.predict.resolve_network', return_value=('ws://x', 'test')), + patch('gittensor.cli.issue_commands.predict.resolve_netuid_from_contract', return_value=1), patch('gittensor.cli.issue_commands.predict.fetch_issue_from_contract', return_value=sample_issue), patch('gittensor.cli.issue_commands.predict.fetch_open_issue_pull_requests', return_value=sample_prs), patch('gittensor.cli.issue_commands.predict._resolve_registered_miner_hotkey') as mock_resolve_miner, diff --git a/tests/validator/conftest.py b/tests/validator/conftest.py index 5201a886..ec946934 100644 --- a/tests/validator/conftest.py +++ b/tests/validator/conftest.py @@ -15,7 +15,7 @@ import pytest from gittensor.classes import PRState, PullRequest -from gittensor.validator.configurations.tier_config import ( +from gittensor.validator.oss_contributions.tier_config import ( TIERS, Tier, TierConfig, diff --git a/gittensor/validator/test/__init__.py b/tests/validator/merge_predictions/__init__.py similarity index 100% rename from gittensor/validator/test/__init__.py rename to tests/validator/merge_predictions/__init__.py diff --git a/tests/validator/merge_predictions/conftest.py b/tests/validator/merge_predictions/conftest.py new file mode 100644 index 00000000..9e4dd65e --- /dev/null +++ b/tests/validator/merge_predictions/conftest.py @@ -0,0 +1,130 @@ +# Entrius 2025 + +"""Shared fixtures for merge predictions tests.""" + +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock + +import pytest +from bittensor.core.synapse import TerminalInfo + +from gittensor.synapses import PredictionSynapse +from gittensor.validator.merge_predictions.mp_storage import PredictionStorage +from gittensor.validator.merge_predictions.scoring import PrOutcome, PrPrediction + +# ============================================================================ +# Storage +# ============================================================================ + + +@pytest.fixture +def mp_storage(tmp_path): + """Real SQLite-backed PredictionStorage in a temp directory.""" + return PredictionStorage(db_path=str(tmp_path / 'test.db')) + + +# ============================================================================ +# Time helpers +# ============================================================================ + + +@pytest.fixture +def base_times(): + """Spread of datetimes across a 30-day window for scoring tests.""" + pr_open = datetime(2025, 6, 1, tzinfo=timezone.utc) + return { + 'pr_open': pr_open, + 'peak_variance': pr_open + timedelta(days=10), + 'prediction_early': pr_open + timedelta(days=2), + 'prediction_mid': pr_open + timedelta(days=15), + 'prediction_late': pr_open + timedelta(days=28), + 'settlement': pr_open + timedelta(days=30), + } + + +# ============================================================================ +# Outcomes & Predictions +# ============================================================================ + + +@pytest.fixture +def sample_outcomes(base_times): + """4 PRs: #1 merged, #2-#4 non-merged.""" + pr_open = base_times['pr_open'] + return [ + PrOutcome(pr_number=1, outcome=1.0, pr_open_time=pr_open), + PrOutcome(pr_number=2, outcome=0.0, pr_open_time=pr_open), + PrOutcome(pr_number=3, outcome=0.0, pr_open_time=pr_open), + PrOutcome(pr_number=4, outcome=0.0, pr_open_time=pr_open), + ] + + +@pytest.fixture +def alice_predictions(base_times): + """Early + accurate miner: high on merged PR, low on others.""" + t = base_times['prediction_early'] + return [ + PrPrediction(pr_number=1, prediction=0.70, prediction_time=t, variance_at_prediction=0.05), + PrPrediction(pr_number=2, prediction=0.15, prediction_time=t, variance_at_prediction=0.05), + PrPrediction(pr_number=3, prediction=0.10, prediction_time=t, variance_at_prediction=0.05), + PrPrediction(pr_number=4, prediction=0.05, prediction_time=t, variance_at_prediction=0.05), + ] + + +@pytest.fixture +def dave_predictions(base_times): + """Spray-and-pray miner: uniform 0.25 across all PRs.""" + t = base_times['prediction_early'] + return [ + PrPrediction(pr_number=1, prediction=0.25, prediction_time=t, variance_at_prediction=0.05), + PrPrediction(pr_number=2, prediction=0.25, prediction_time=t, variance_at_prediction=0.05), + PrPrediction(pr_number=3, prediction=0.25, prediction_time=t, variance_at_prediction=0.05), + PrPrediction(pr_number=4, prediction=0.25, prediction_time=t, variance_at_prediction=0.05), + ] + + +# ============================================================================ +# Validator mock +# ============================================================================ + + +@pytest.fixture +def mock_validator(mp_storage): + """MagicMock validator with mp_storage, metagraph, and subtensor.""" + v = MagicMock() + v.mp_storage = mp_storage + + # metagraph with 3 registered hotkeys + v.metagraph.hotkeys = ['hk_alice', 'hk_bob', 'hk_charlie'] + v.metagraph.S = [100.0, 50.0, 25.0] + + v.subtensor = MagicMock() + return v + + +# ============================================================================ +# Synapse factory +# ============================================================================ + + +@pytest.fixture +def make_synapse(): + """Factory that builds a PredictionSynapse with configurable fields.""" + + def _make( + predictions=None, + issue_id=1, + repository='test/repo', + github_access_token='ghp_test123', + hotkey='hk_alice', + ): + synapse = PredictionSynapse( + predictions=predictions or {1: 0.5}, + issue_id=issue_id, + repository=repository, + github_access_token=github_access_token, + ) + synapse.dendrite = TerminalInfo(hotkey=hotkey) + return synapse + + return _make diff --git a/tests/validator/merge_predictions/test_merge_predictions.py b/tests/validator/merge_predictions/test_merge_predictions.py new file mode 100644 index 00000000..c58fa49f --- /dev/null +++ b/tests/validator/merge_predictions/test_merge_predictions.py @@ -0,0 +1,865 @@ +# Entrius 2025 + +"""Merge predictions test suite. + +Covers: storage, handler, scoring, validation, and settlement. + +Run: + pytest tests/validator/merge_predictions/ -v +""" + +import asyncio +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock, patch + +import pytest + +from gittensor.constants import ( + PREDICTIONS_COOLDOWN_SECONDS, + PREDICTIONS_CORRECTNESS_EXPONENT, + PREDICTIONS_EMA_BETA, + PREDICTIONS_MAX_CONSENSUS_BONUS, + PREDICTIONS_MAX_ORDER_BONUS, + PREDICTIONS_MAX_TIMELINESS_BONUS, + PREDICTIONS_TIMELINESS_EXPONENT, +) +from gittensor.validator.merge_predictions.scoring import ( + MinerIssueScore, + PrPrediction, + compute_merged_pr_order_ranks, + score_consensus_bonus, + score_correctness, + score_miner_issue, + score_order_bonus, + score_timeliness, + update_ema, +) +from gittensor.validator.merge_predictions.validation import validate_prediction_values + + +def _run(coro): + """Run an async coroutine synchronously (no pytest-asyncio needed).""" + return asyncio.run(coro) + + +# ============================================================================= +# 1. Storage +# ============================================================================= + + +class TestPredictionStorage: + """Tests for PredictionStorage (real SQLite, no mocking).""" + + def test_tables_created(self, mp_storage): + with mp_storage._get_connection() as conn: + tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()} + assert 'predictions' in tables + assert 'prediction_emas' in tables + assert 'settled_issues' in tables + + def test_store_and_retrieve_prediction(self, mp_storage): + mp_storage.store_prediction( + uid=0, + hotkey='hk', + github_id='gh1', + issue_id=1, + repository='r/r', + pr_number=10, + prediction=0.6, + variance_at_prediction=0.1, + ) + rows = mp_storage.get_predictions_for_issue(1) + assert len(rows) == 1 + assert rows[0]['prediction'] == pytest.approx(0.6) + assert rows[0]['pr_number'] == 10 + + def test_upsert_replaces_prediction(self, mp_storage): + kwargs = dict(uid=0, hotkey='hk', github_id='gh1', issue_id=1, repository='r/r', pr_number=10) + mp_storage.store_prediction(**kwargs, prediction=0.3, variance_at_prediction=0.1) + mp_storage.store_prediction(**kwargs, prediction=0.8, variance_at_prediction=0.2) + rows = mp_storage.get_predictions_for_issue(1) + assert len(rows) == 1 + assert rows[0]['prediction'] == pytest.approx(0.8) + + def test_upsert_preserves_other_prs(self, mp_storage): + base = dict(uid=0, hotkey='hk', github_id='gh1', issue_id=1, repository='r/r') + mp_storage.store_prediction(**base, pr_number=1, prediction=0.3, variance_at_prediction=0.0) + mp_storage.store_prediction(**base, pr_number=2, prediction=0.4, variance_at_prediction=0.0) + + # Update only PR #1 + mp_storage.store_prediction(**base, pr_number=1, prediction=0.5, variance_at_prediction=0.0) + + rows = mp_storage.get_predictions_for_issue(1) + by_pr = {r['pr_number']: r for r in rows} + assert by_pr[1]['prediction'] == pytest.approx(0.5) + assert by_pr[2]['prediction'] == pytest.approx(0.4) + + def test_miner_total_for_issue(self, mp_storage): + base = dict(uid=0, hotkey='hk', github_id='gh1', issue_id=1, repository='r/r') + mp_storage.store_prediction(**base, pr_number=1, prediction=0.3, variance_at_prediction=0.0) + mp_storage.store_prediction(**base, pr_number=2, prediction=0.4, variance_at_prediction=0.0) + total = mp_storage.get_miner_total_for_issue(0, 'hk', 1) + assert total == pytest.approx(0.7) + + def test_miner_total_excludes_prs(self, mp_storage): + base = dict(uid=0, hotkey='hk', github_id='gh1', issue_id=1, repository='r/r') + mp_storage.store_prediction(**base, pr_number=1, prediction=0.3, variance_at_prediction=0.0) + mp_storage.store_prediction(**base, pr_number=2, prediction=0.4, variance_at_prediction=0.0) + mp_storage.store_prediction(**base, pr_number=3, prediction=0.2, variance_at_prediction=0.0) + total = mp_storage.get_miner_total_for_issue(0, 'hk', 1, exclude_prs={2, 3}) + assert total == pytest.approx(0.3) + + def test_cooldown_active(self, mp_storage): + mp_storage.store_prediction( + uid=0, + hotkey='hk', + github_id='gh1', + issue_id=1, + repository='r/r', + pr_number=1, + prediction=0.5, + variance_at_prediction=0.0, + ) + remaining = mp_storage.check_cooldown(0, 'hk', 1, 1) + assert remaining is not None + assert remaining > 0 + + def test_cooldown_expired(self, mp_storage): + """Store a prediction with a timestamp far in the past, then verify cooldown is None.""" + # Insert directly with an old timestamp to avoid patching datetime + old_ts = (datetime.now(timezone.utc) - timedelta(seconds=PREDICTIONS_COOLDOWN_SECONDS + 60)).isoformat() + with mp_storage._get_connection() as conn: + conn.execute( + 'INSERT INTO predictions (uid, hotkey, github_id, issue_id, repository, pr_number, prediction, timestamp, variance_at_prediction) ' + 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', + (0, 'hk', 'gh1', 1, 'r/r', 1, 0.5, old_ts, 0.0), + ) + conn.commit() + + remaining = mp_storage.check_cooldown(0, 'hk', 1, 1) + assert remaining is None + + def test_cooldown_no_prior_prediction(self, mp_storage): + assert mp_storage.check_cooldown(0, 'hk', 1, 1) is None + + def test_compute_variance_single_miner(self, mp_storage): + mp_storage.store_prediction( + uid=0, + hotkey='hk', + github_id='gh1', + issue_id=1, + repository='r/r', + pr_number=1, + prediction=0.5, + variance_at_prediction=0.0, + ) + assert mp_storage.compute_current_variance(1) == pytest.approx(0.0) + + def test_compute_variance_disagreement(self, mp_storage): + base = dict(github_id='gh1', issue_id=1, repository='r/r', pr_number=1) + mp_storage.store_prediction(uid=0, hotkey='hk0', **base, prediction=0.9, variance_at_prediction=0.0) + mp_storage.store_prediction(uid=1, hotkey='hk1', **base, prediction=0.1, variance_at_prediction=0.0) + var = mp_storage.compute_current_variance(1) + # var((0.9,0.1)) = mean(x^2) - mean(x)^2 = 0.41 - 0.25 = 0.16 + assert var > 0 + + def test_peak_variance_time(self, mp_storage): + base = dict(uid=0, hotkey='hk', github_id='gh1', issue_id=1, repository='r/r') + mp_storage.store_prediction(**base, pr_number=1, prediction=0.5, variance_at_prediction=0.1) + mp_storage.store_prediction(**base, pr_number=2, prediction=0.5, variance_at_prediction=0.9) + peak = mp_storage.get_peak_variance_time(1) + assert peak is not None + + def test_ema_default_zero(self, mp_storage): + assert mp_storage.get_ema('unknown_github_id') == 0.0 + + def test_ema_upsert_increments_rounds(self, mp_storage): + mp_storage.update_ema('gh1', 0.5) + mp_storage.update_ema('gh1', 0.6) + emas = mp_storage.get_all_emas() + by_id = {e['github_id']: e for e in emas} + assert by_id['gh1']['rounds'] == 2 + + def test_get_all_emas(self, mp_storage): + mp_storage.update_ema('gh1', 0.5) + mp_storage.update_ema('gh2', 0.3) + emas = mp_storage.get_all_emas() + ids = {e['github_id'] for e in emas} + assert ids == {'gh1', 'gh2'} + + def test_delete_predictions_for_issue(self, mp_storage): + base = dict(uid=0, hotkey='hk', github_id='gh1', issue_id=1, repository='r/r') + mp_storage.store_prediction(**base, pr_number=1, prediction=0.3, variance_at_prediction=0.0) + mp_storage.store_prediction(**base, pr_number=2, prediction=0.4, variance_at_prediction=0.0) + deleted = mp_storage.delete_predictions_for_issue(1) + assert deleted == 2 + assert mp_storage.get_predictions_for_issue(1) == [] + + def test_delete_predictions_no_rows(self, mp_storage): + deleted = mp_storage.delete_predictions_for_issue(999) + assert deleted == 0 + + def test_mark_and_check_settled(self, mp_storage): + mp_storage.mark_issue_settled(42, 'scored', merged_pr_number=7) + assert mp_storage.is_issue_settled(42) is True + + def test_is_issue_settled_false(self, mp_storage): + assert mp_storage.is_issue_settled(999) is False + + def test_mark_settled_voided(self, mp_storage): + mp_storage.mark_issue_settled(10, 'voided') + assert mp_storage.is_issue_settled(10) is True + + def test_mark_settled_idempotent(self, mp_storage): + mp_storage.mark_issue_settled(42, 'scored', merged_pr_number=7) + mp_storage.mark_issue_settled(42, 'scored', merged_pr_number=7) + assert mp_storage.is_issue_settled(42) is True + + +# ============================================================================= +# 2. Handler +# ============================================================================= + + +class TestPredictionHandler: + """Tests for handle_prediction, blacklist_prediction, priority_prediction.""" + + @patch('gittensor.validator.merge_predictions.handler.validate_prediction_values', return_value=None) + @patch('gittensor.validator.merge_predictions.handler.validate_github_credentials', return_value=('gh_alice', None)) + @patch('gittensor.validator.merge_predictions.handler.check_prs_open', return_value=(None, {1})) + @patch( + 'gittensor.validator.merge_predictions.handler.check_issue_active', + return_value=(None, MagicMock(issue_number=10)), + ) + def test_successful_prediction_stored(self, _cia, _cpo, _vgc, _vpv, mock_validator, make_synapse): + from gittensor.validator.merge_predictions.handler import handle_prediction + + synapse = make_synapse(predictions={1: 0.5}, hotkey='hk_alice') + result = _run(handle_prediction(mock_validator, synapse)) + assert result.accepted is True + rows = mock_validator.mp_storage.get_predictions_for_issue(1) + assert len(rows) == 1 + + @patch('gittensor.validator.merge_predictions.handler.check_issue_active', return_value=('Issue not found', None)) + def test_reject_inactive_issue(self, _cia, mock_validator, make_synapse): + from gittensor.validator.merge_predictions.handler import handle_prediction + + synapse = make_synapse(hotkey='hk_alice') + result = _run(handle_prediction(mock_validator, synapse)) + assert result.accepted is False + assert 'Issue not found' in result.rejection_reason + + @patch('gittensor.validator.merge_predictions.handler.check_prs_open', return_value=('PR #1 is not open', set())) + @patch( + 'gittensor.validator.merge_predictions.handler.check_issue_active', + return_value=(None, MagicMock(issue_number=10)), + ) + def test_reject_closed_pr(self, _cia, _cpo, mock_validator, make_synapse): + from gittensor.validator.merge_predictions.handler import handle_prediction + + synapse = make_synapse(hotkey='hk_alice') + result = _run(handle_prediction(mock_validator, synapse)) + assert result.accepted is False + assert 'not open' in result.rejection_reason + + @patch('gittensor.validator.merge_predictions.handler.validate_github_credentials', return_value=(None, 'Bad PAT')) + @patch('gittensor.validator.merge_predictions.handler.check_prs_open', return_value=(None, {1})) + @patch( + 'gittensor.validator.merge_predictions.handler.check_issue_active', + return_value=(None, MagicMock(issue_number=10)), + ) + def test_reject_invalid_github_creds(self, _cia, _cpo, _vgc, mock_validator, make_synapse): + from gittensor.validator.merge_predictions.handler import handle_prediction + + synapse = make_synapse(hotkey='hk_alice') + result = _run(handle_prediction(mock_validator, synapse)) + assert result.accepted is False + assert 'Bad PAT' in result.rejection_reason + + @patch('gittensor.validator.merge_predictions.handler.validate_prediction_values', return_value='Values bad') + @patch('gittensor.validator.merge_predictions.handler.validate_github_credentials', return_value=('gh_alice', None)) + @patch('gittensor.validator.merge_predictions.handler.check_prs_open', return_value=(None, {1})) + @patch( + 'gittensor.validator.merge_predictions.handler.check_issue_active', + return_value=(None, MagicMock(issue_number=10)), + ) + def test_reject_invalid_values(self, _cia, _cpo, _vgc, _vpv, mock_validator, make_synapse): + from gittensor.validator.merge_predictions.handler import handle_prediction + + synapse = make_synapse(hotkey='hk_alice') + result = _run(handle_prediction(mock_validator, synapse)) + assert result.accepted is False + assert 'Values bad' in result.rejection_reason + + @patch('gittensor.validator.merge_predictions.handler.validate_prediction_values', return_value=None) + @patch('gittensor.validator.merge_predictions.handler.validate_github_credentials', return_value=('gh_alice', None)) + @patch('gittensor.validator.merge_predictions.handler.check_prs_open', return_value=(None, {1})) + @patch( + 'gittensor.validator.merge_predictions.handler.check_issue_active', + return_value=(None, MagicMock(issue_number=10)), + ) + def test_reject_cooldown(self, _cia, _cpo, _vgc, _vpv, mock_validator, make_synapse): + from gittensor.validator.merge_predictions.handler import handle_prediction + + # First prediction succeeds + s1 = make_synapse(predictions={1: 0.3}, hotkey='hk_alice') + _run(handle_prediction(mock_validator, s1)) + + # Immediate re-prediction hits cooldown + s2 = make_synapse(predictions={1: 0.4}, hotkey='hk_alice') + result = _run(handle_prediction(mock_validator, s2)) + assert result.accepted is False + assert 'cooldown' in result.rejection_reason + + @patch('gittensor.validator.merge_predictions.handler.validate_prediction_values', return_value=None) + @patch('gittensor.validator.merge_predictions.handler.validate_github_credentials', return_value=('gh_alice', None)) + @patch('gittensor.validator.merge_predictions.handler.check_prs_open', return_value=(None, {1, 2})) + @patch( + 'gittensor.validator.merge_predictions.handler.check_issue_active', + return_value=(None, MagicMock(issue_number=10)), + ) + def test_reject_total_exceeds_one(self, _cia, _cpo, _vgc, _vpv, mock_validator, make_synapse): + from gittensor.validator.merge_predictions.handler import handle_prediction + + # Seed existing prediction via storage directly to avoid cooldown + mock_validator.mp_storage.store_prediction( + uid=0, + hotkey='hk_alice', + github_id='gh_alice', + issue_id=1, + repository='test/repo', + pr_number=1, + prediction=0.8, + variance_at_prediction=0.0, + ) + + # New prediction on different PR would push total > 1.0 + s = make_synapse(predictions={2: 0.5}, hotkey='hk_alice') + result = _run(handle_prediction(mock_validator, s)) + assert result.accepted is False + assert 'exceeds 1.0' in result.rejection_reason + + def test_blacklist_unregistered_hotkey(self, mock_validator, make_synapse): + from gittensor.validator.merge_predictions.handler import blacklist_prediction + + synapse = make_synapse(hotkey='hk_unknown') + is_blacklisted, reason = _run(blacklist_prediction(mock_validator, synapse)) + assert is_blacklisted is True + assert 'Unregistered' in reason + + def test_blacklist_allows_registered(self, mock_validator, make_synapse): + from gittensor.validator.merge_predictions.handler import blacklist_prediction + + synapse = make_synapse(hotkey='hk_alice') + is_blacklisted, _ = _run(blacklist_prediction(mock_validator, synapse)) + assert is_blacklisted is False + + def test_priority_by_stake(self, mock_validator, make_synapse): + from gittensor.validator.merge_predictions.handler import priority_prediction + + synapse = make_synapse(hotkey='hk_alice') + priority = _run(priority_prediction(mock_validator, synapse)) + assert priority == pytest.approx(100.0) + + +# ============================================================================= +# 3. Scoring +# ============================================================================= + + +class TestPredictionScoring: + """Pure function tests for scoring math.""" + + # -- Correctness -- + + def test_correctness_merged_pr(self): + result = score_correctness(0.9, 1.0) + assert result == pytest.approx(0.9**PREDICTIONS_CORRECTNESS_EXPONENT) + + def test_correctness_non_merged_pr(self): + result = score_correctness(0.1, 0.0) + assert result == pytest.approx(0.9**PREDICTIONS_CORRECTNESS_EXPONENT) + + def test_correctness_wrong_prediction(self): + result = score_correctness(0.3, 1.0) + assert result == pytest.approx(0.3**PREDICTIONS_CORRECTNESS_EXPONENT) + + def test_correctness_uniform_spray(self): + result = score_correctness(0.25, 1.0) + assert result == pytest.approx(0.25**PREDICTIONS_CORRECTNESS_EXPONENT) + + # -- Timeliness -- + + def test_timeliness_at_pr_open(self, base_times): + result = score_timeliness(base_times['pr_open'], base_times['settlement'], base_times['pr_open']) + assert result == pytest.approx(PREDICTIONS_MAX_TIMELINESS_BONUS) + + def test_timeliness_at_settlement(self, base_times): + result = score_timeliness(base_times['settlement'], base_times['settlement'], base_times['pr_open']) + assert result == pytest.approx(0.0) + + def test_timeliness_midpoint(self, base_times): + midpoint = base_times['pr_open'] + timedelta(days=15) + result = score_timeliness(midpoint, base_times['settlement'], base_times['pr_open']) + expected = PREDICTIONS_MAX_TIMELINESS_BONUS * (0.5**PREDICTIONS_TIMELINESS_EXPONENT) + assert result == pytest.approx(expected) + + def test_timeliness_zero_window(self): + t = datetime(2025, 6, 1, tzinfo=timezone.utc) + assert score_timeliness(t, t, t) == 0.0 + + # -- Consensus -- + + def test_consensus_before_peak(self, base_times): + result = score_consensus_bonus( + base_times['prediction_early'], base_times['peak_variance'], base_times['settlement'] + ) + assert result == pytest.approx(PREDICTIONS_MAX_CONSENSUS_BONUS) + + def test_consensus_at_peak(self, base_times): + result = score_consensus_bonus( + base_times['peak_variance'], base_times['peak_variance'], base_times['settlement'] + ) + assert result == pytest.approx(PREDICTIONS_MAX_CONSENSUS_BONUS) + + def test_consensus_after_peak_midway(self, base_times): + peak = base_times['peak_variance'] + settle = base_times['settlement'] + mid = peak + (settle - peak) / 2 + result = score_consensus_bonus(mid, peak, settle) + assert result == pytest.approx(PREDICTIONS_MAX_CONSENSUS_BONUS * 0.5) + + def test_consensus_at_settlement(self, base_times): + result = score_consensus_bonus(base_times['settlement'], base_times['peak_variance'], base_times['settlement']) + assert result == pytest.approx(0.0) + + # -- Order -- + + def test_order_rank_1(self): + assert score_order_bonus(1) == pytest.approx(PREDICTIONS_MAX_ORDER_BONUS) + + def test_order_rank_2(self): + assert score_order_bonus(2) == pytest.approx(PREDICTIONS_MAX_ORDER_BONUS / 2) + + def test_order_rank_0_unqualified(self): + assert score_order_bonus(0) == 0.0 + + def test_compute_order_ranks_filters_below_threshold(self): + preds = { + 0: [ + PrPrediction( + pr_number=1, + prediction=0.5, + prediction_time=datetime(2025, 6, 1, tzinfo=timezone.utc), + variance_at_prediction=0.0, + ) + ], + 1: [ + PrPrediction( + pr_number=1, + prediction=0.9, + prediction_time=datetime(2025, 6, 2, tzinfo=timezone.utc), + variance_at_prediction=0.0, + ) + ], + } + ranks = compute_merged_pr_order_ranks(preds, merged_pr_number=1) + assert 0 not in ranks + assert ranks[1] == 1 + + def test_compute_order_ranks_sorts_by_time(self): + t1 = datetime(2025, 6, 1, tzinfo=timezone.utc) + t2 = datetime(2025, 6, 2, tzinfo=timezone.utc) + preds = { + 0: [PrPrediction(pr_number=1, prediction=0.9, prediction_time=t2, variance_at_prediction=0.0)], + 1: [PrPrediction(pr_number=1, prediction=0.8, prediction_time=t1, variance_at_prediction=0.0)], + } + ranks = compute_merged_pr_order_ranks(preds, merged_pr_number=1) + assert ranks[1] == 1 # earlier + assert ranks[0] == 2 + + # -- Aggregation: score_miner_issue -- + + def test_score_miner_issue_weighted_mean(self, base_times, sample_outcomes): + """Merged PR gets weight=N in the issue score (N = total PRs).""" + t = base_times['prediction_early'] + preds = [ + PrPrediction(pr_number=1, prediction=0.9, prediction_time=t, variance_at_prediction=0.05), + PrPrediction(pr_number=2, prediction=0.05, prediction_time=t, variance_at_prediction=0.05), + PrPrediction(pr_number=3, prediction=0.03, prediction_time=t, variance_at_prediction=0.05), + PrPrediction(pr_number=4, prediction=0.02, prediction_time=t, variance_at_prediction=0.05), + ] + result = score_miner_issue( + uid=0, + predictions=preds, + outcomes=sample_outcomes, + settlement_time=base_times['settlement'], + peak_variance_time=base_times['peak_variance'], + merged_pr_order_ranks={0: 1}, + ) + assert isinstance(result, MinerIssueScore) + assert result.issue_score > 0 + merged_score = next(ps for ps in result.pr_scores if ps.pr_number == 1) + assert merged_score.score > 0 + + # -- EMA -- + + def test_update_ema(self): + result = update_ema(current_round_score=1.0, previous_ema=0.0) + expected = PREDICTIONS_EMA_BETA * 1.0 + (1.0 - PREDICTIONS_EMA_BETA) * 0.0 + assert result == pytest.approx(expected) + + +# ============================================================================= +# 4. Validation +# ============================================================================= + + +class TestValidation: + """Pure function tests for validate_prediction_values.""" + + def test_valid_predictions(self): + assert validate_prediction_values({1: 0.5, 2: 0.3}) is None + + def test_empty_predictions(self): + result = validate_prediction_values({}) + assert result is not None + assert 'Empty' in result + + def test_negative_pr_number(self): + result = validate_prediction_values({-1: 0.5}) + assert result is not None + assert 'Invalid PR number' in result + + def test_value_out_of_range(self): + result = validate_prediction_values({1: 1.5}) + assert result is not None + assert 'out of range' in result + + def test_total_exceeds_one(self): + result = validate_prediction_values({1: 0.6, 2: 0.5}) + assert result is not None + assert 'exceeds 1.0' in result + + +# ============================================================================= +# 5. Settlement +# ============================================================================= + + +class TestSettlement: + """Tests for merge_predictions() settlement orchestrator. + + Settlement now queries COMPLETED and CANCELLED issues from the contract + (not ACTIVE). Predictions are deleted after settlement as the "settled" marker. + """ + + def _seed_predictions(self, mp_storage, uid, hotkey, github_id, issue_id, preds): + """Helper: store a set of predictions for a miner.""" + for pr_num, value in preds.items(): + mp_storage.store_prediction( + uid=uid, + hotkey=hotkey, + github_id=github_id, + issue_id=issue_id, + repository='test/repo', + pr_number=pr_num, + prediction=value, + variance_at_prediction=0.05, + ) + + def _make_contract_issue(self, issue_id=1, repo='test/repo', issue_number=10): + issue = MagicMock() + issue.id = issue_id + issue.repository_full_name = repo + issue.issue_number = issue_number + return issue + + def _setup_contract_mock(self, MockContract, completed=None, cancelled=None): + """Configure the contract mock to return different issues per status.""" + from gittensor.validator.issue_competitions.contract_client import IssueStatus + + def get_issues_side_effect(status): + if status == IssueStatus.COMPLETED: + return completed or [] + elif status == IssueStatus.CANCELLED: + return cancelled or [] + return [] + + MockContract.return_value.get_issues_by_status.side_effect = get_issues_side_effect + + @patch('gittensor.validator.merge_predictions.settlement.get_pr_open_times') + @patch('gittensor.validator.merge_predictions.settlement.check_github_issue_closed') + @patch('gittensor.validator.merge_predictions.settlement.get_contract_address', return_value='5Faddr') + @patch('gittensor.validator.merge_predictions.settlement.GITTENSOR_VALIDATOR_PAT', 'ghp_test') + @patch('gittensor.validator.merge_predictions.settlement.IssueCompetitionContractClient') + def test_settle_completed_issue_updates_ema( + self, MockContract, _gca, mock_check_closed, mock_pr_times, mock_validator + ): + from gittensor.validator.merge_predictions.settlement import merge_predictions + + pr_open_time = datetime(2025, 6, 1, tzinfo=timezone.utc) + + contract_issue = self._make_contract_issue() + self._setup_contract_mock(MockContract, completed=[contract_issue]) + + mock_check_closed.return_value = {'is_closed': True, 'pr_number': 1} + mock_pr_times.return_value = {1: pr_open_time, 2: pr_open_time} + + self._seed_predictions( + mock_validator.mp_storage, + uid=0, + hotkey='hk_alice', + github_id='gh_alice', + issue_id=1, + preds={1: 0.7, 2: 0.2}, + ) + + _run(merge_predictions(mock_validator, {})) + + ema = mock_validator.mp_storage.get_ema('gh_alice') + assert ema > 0 + # Predictions deleted after settlement + assert mock_validator.mp_storage.get_predictions_for_issue(1) == [] + + @patch('gittensor.validator.merge_predictions.settlement.get_pr_open_times') + @patch('gittensor.validator.merge_predictions.settlement.check_github_issue_closed') + @patch('gittensor.validator.merge_predictions.settlement.get_contract_address', return_value='5Faddr') + @patch('gittensor.validator.merge_predictions.settlement.GITTENSOR_VALIDATOR_PAT', 'ghp_test') + @patch('gittensor.validator.merge_predictions.settlement.IssueCompetitionContractClient') + def test_settle_multiple_completed_issues( + self, MockContract, _gca, mock_check_closed, mock_pr_times, mock_validator + ): + from gittensor.validator.merge_predictions.settlement import merge_predictions + + pr_open_time = datetime(2025, 6, 1, tzinfo=timezone.utc) + + issue1 = self._make_contract_issue(issue_id=1, issue_number=10) + issue2 = self._make_contract_issue(issue_id=2, issue_number=20) + self._setup_contract_mock(MockContract, completed=[issue1, issue2]) + + mock_check_closed.return_value = {'is_closed': True, 'pr_number': 1} + mock_pr_times.return_value = {1: pr_open_time} + + self._seed_predictions( + mock_validator.mp_storage, uid=0, hotkey='hk_alice', github_id='gh_alice', issue_id=1, preds={1: 0.8} + ) + self._seed_predictions( + mock_validator.mp_storage, uid=0, hotkey='hk_alice', github_id='gh_alice', issue_id=2, preds={1: 0.9} + ) + + _run(merge_predictions(mock_validator, {})) + + emas = mock_validator.mp_storage.get_all_emas() + gh_alice = next(e for e in emas if e['github_id'] == 'gh_alice') + assert gh_alice['rounds'] == 2 + # Both issues' predictions deleted + assert mock_validator.mp_storage.get_predictions_for_issue(1) == [] + assert mock_validator.mp_storage.get_predictions_for_issue(2) == [] + + @patch('gittensor.validator.merge_predictions.settlement.check_github_issue_closed') + @patch('gittensor.validator.merge_predictions.settlement.get_contract_address', return_value='5Faddr') + @patch('gittensor.validator.merge_predictions.settlement.GITTENSOR_VALIDATOR_PAT', 'ghp_test') + @patch('gittensor.validator.merge_predictions.settlement.IssueCompetitionContractClient') + def test_cancelled_issue_no_merge_no_ema_impact(self, MockContract, _gca, mock_check_closed, mock_validator): + """Cancelled issue with no merged PR: predictions voided, no EMA impact.""" + from gittensor.validator.merge_predictions.settlement import merge_predictions + + contract_issue = self._make_contract_issue() + self._setup_contract_mock(MockContract, cancelled=[contract_issue]) + + mock_check_closed.return_value = {'is_closed': True, 'pr_number': None} + + self._seed_predictions( + mock_validator.mp_storage, uid=0, hotkey='hk_alice', github_id='gh_alice', issue_id=1, preds={1: 0.8} + ) + + _run(merge_predictions(mock_validator, {})) + + assert mock_validator.mp_storage.get_ema('gh_alice') == 0.0 + # Predictions deleted even though voided + assert mock_validator.mp_storage.get_predictions_for_issue(1) == [] + + @patch('gittensor.validator.merge_predictions.settlement.get_pr_open_times') + @patch('gittensor.validator.merge_predictions.settlement.check_github_issue_closed') + @patch('gittensor.validator.merge_predictions.settlement.get_contract_address', return_value='5Faddr') + @patch('gittensor.validator.merge_predictions.settlement.GITTENSOR_VALIDATOR_PAT', 'ghp_test') + @patch('gittensor.validator.merge_predictions.settlement.IssueCompetitionContractClient') + def test_cancelled_issue_with_merge_still_scored( + self, MockContract, _gca, mock_check_closed, mock_pr_times, mock_validator + ): + """Cancelled but PR was merged (solver not in subnet): predictions still scored.""" + from gittensor.validator.merge_predictions.settlement import merge_predictions + + pr_open_time = datetime(2025, 6, 1, tzinfo=timezone.utc) + + contract_issue = self._make_contract_issue() + self._setup_contract_mock(MockContract, cancelled=[contract_issue]) + + mock_check_closed.return_value = {'is_closed': True, 'pr_number': 1} + mock_pr_times.return_value = {1: pr_open_time, 2: pr_open_time} + + self._seed_predictions( + mock_validator.mp_storage, + uid=0, + hotkey='hk_alice', + github_id='gh_alice', + issue_id=1, + preds={1: 0.7, 2: 0.2}, + ) + + _run(merge_predictions(mock_validator, {})) + + ema = mock_validator.mp_storage.get_ema('gh_alice') + assert ema > 0 + # Predictions deleted after scoring + assert mock_validator.mp_storage.get_predictions_for_issue(1) == [] + + @patch('gittensor.validator.merge_predictions.settlement.get_contract_address', return_value='5Faddr') + @patch('gittensor.validator.merge_predictions.settlement.GITTENSOR_VALIDATOR_PAT', 'ghp_test') + @patch('gittensor.validator.merge_predictions.settlement.IssueCompetitionContractClient') + def test_already_settled_skipped(self, MockContract, _gca, mock_validator): + """Already-settled issues are skipped without calling GitHub, even if predictions exist.""" + from gittensor.validator.merge_predictions.settlement import merge_predictions + + contract_issue = self._make_contract_issue() + self._setup_contract_mock(MockContract, completed=[contract_issue]) + + # Pre-mark as settled and seed predictions anyway + mock_validator.mp_storage.mark_issue_settled(contract_issue.id, 'scored', merged_pr_number=1) + self._seed_predictions( + mock_validator.mp_storage, uid=0, hotkey='hk_alice', github_id='gh_alice', issue_id=1, preds={1: 0.8} + ) + + with patch('gittensor.validator.merge_predictions.settlement.check_github_issue_closed') as mock_check: + _run(merge_predictions(mock_validator, {})) + # GitHub should NOT be called since issue is already settled + mock_check.assert_not_called() + + # Predictions should be untouched (not deleted by settlement) + assert len(mock_validator.mp_storage.get_predictions_for_issue(1)) == 1 + + @patch('gittensor.validator.merge_predictions.settlement.get_pr_open_times') + @patch('gittensor.validator.merge_predictions.settlement.check_github_issue_closed') + @patch('gittensor.validator.merge_predictions.settlement.get_contract_address', return_value='5Faddr') + @patch('gittensor.validator.merge_predictions.settlement.GITTENSOR_VALIDATOR_PAT', 'ghp_test') + @patch('gittensor.validator.merge_predictions.settlement.IssueCompetitionContractClient') + def test_deregistered_miner_skipped(self, MockContract, _gca, mock_check_closed, mock_pr_times, mock_validator): + from gittensor.validator.merge_predictions.settlement import merge_predictions + + pr_open_time = datetime(2025, 6, 1, tzinfo=timezone.utc) + contract_issue = self._make_contract_issue() + self._setup_contract_mock(MockContract, completed=[contract_issue]) + + mock_check_closed.return_value = {'is_closed': True, 'pr_number': 1} + mock_pr_times.return_value = {1: pr_open_time} + + self._seed_predictions( + mock_validator.mp_storage, uid=0, hotkey='hk_alice', github_id='gh_alice', issue_id=1, preds={1: 0.7} + ) + self._seed_predictions( + mock_validator.mp_storage, uid=5, hotkey='hk_gone', github_id='gh_gone', issue_id=1, preds={1: 0.6} + ) + + _run(merge_predictions(mock_validator, {})) + + assert mock_validator.mp_storage.get_ema('gh_alice') > 0 + assert mock_validator.mp_storage.get_ema('gh_gone') == 0.0 + # Predictions deleted for all miners (including deregistered) + assert mock_validator.mp_storage.get_predictions_for_issue(1) == [] + + @patch('gittensor.validator.merge_predictions.settlement.get_pr_open_times') + @patch('gittensor.validator.merge_predictions.settlement.check_github_issue_closed') + @patch('gittensor.validator.merge_predictions.settlement.get_contract_address', return_value='5Faddr') + @patch('gittensor.validator.merge_predictions.settlement.GITTENSOR_VALIDATOR_PAT', 'ghp_test') + @patch('gittensor.validator.merge_predictions.settlement.IssueCompetitionContractClient') + def test_ema_persists_across_settlements( + self, MockContract, _gca, mock_check_closed, mock_pr_times, mock_validator + ): + from gittensor.validator.merge_predictions.settlement import merge_predictions + + pr_open_time = datetime(2025, 6, 1, tzinfo=timezone.utc) + + # First settlement + issue1 = self._make_contract_issue(issue_id=1, issue_number=10) + self._setup_contract_mock(MockContract, completed=[issue1]) + mock_check_closed.return_value = {'is_closed': True, 'pr_number': 1} + mock_pr_times.return_value = {1: pr_open_time} + + self._seed_predictions( + mock_validator.mp_storage, uid=0, hotkey='hk_alice', github_id='gh_alice', issue_id=1, preds={1: 0.8} + ) + + _run(merge_predictions(mock_validator, {})) + ema_after_first = mock_validator.mp_storage.get_ema('gh_alice') + assert ema_after_first > 0 + assert mock_validator.mp_storage.get_predictions_for_issue(1) == [] + + # Second settlement with a new issue + issue2 = self._make_contract_issue(issue_id=2, issue_number=20) + self._setup_contract_mock(MockContract, completed=[issue2]) + mock_pr_times.return_value = {1: pr_open_time} + + self._seed_predictions( + mock_validator.mp_storage, uid=0, hotkey='hk_alice', github_id='gh_alice', issue_id=2, preds={1: 0.9} + ) + + _run(merge_predictions(mock_validator, {})) + ema_after_second = mock_validator.mp_storage.get_ema('gh_alice') + + assert ema_after_second != ema_after_first + + @patch('gittensor.validator.merge_predictions.settlement.get_pr_open_times') + @patch('gittensor.validator.merge_predictions.settlement.check_github_issue_closed') + @patch('gittensor.validator.merge_predictions.settlement.get_contract_address', return_value='5Faddr') + @patch('gittensor.validator.merge_predictions.settlement.GITTENSOR_VALIDATOR_PAT', 'ghp_test') + @patch('gittensor.validator.merge_predictions.settlement.IssueCompetitionContractClient') + def test_settled_issue_recorded_after_scoring( + self, MockContract, _gca, mock_check_closed, mock_pr_times, mock_validator + ): + """After completed settlement, issue is recorded in settled_issues.""" + from gittensor.validator.merge_predictions.settlement import merge_predictions + + pr_open_time = datetime(2025, 6, 1, tzinfo=timezone.utc) + contract_issue = self._make_contract_issue() + self._setup_contract_mock(MockContract, completed=[contract_issue]) + + mock_check_closed.return_value = {'is_closed': True, 'pr_number': 1} + mock_pr_times.return_value = {1: pr_open_time} + + self._seed_predictions( + mock_validator.mp_storage, uid=0, hotkey='hk_alice', github_id='gh_alice', issue_id=1, preds={1: 0.8} + ) + + _run(merge_predictions(mock_validator, {})) + + assert mock_validator.mp_storage.is_issue_settled(1) is True + + @patch('gittensor.validator.merge_predictions.settlement.check_github_issue_closed') + @patch('gittensor.validator.merge_predictions.settlement.get_contract_address', return_value='5Faddr') + @patch('gittensor.validator.merge_predictions.settlement.GITTENSOR_VALIDATOR_PAT', 'ghp_test') + @patch('gittensor.validator.merge_predictions.settlement.IssueCompetitionContractClient') + def test_voided_issue_recorded(self, MockContract, _gca, mock_check_closed, mock_validator): + """After voiding a cancelled issue, it is recorded in settled_issues.""" + from gittensor.validator.merge_predictions.settlement import merge_predictions + + contract_issue = self._make_contract_issue() + self._setup_contract_mock(MockContract, cancelled=[contract_issue]) + + mock_check_closed.return_value = {'is_closed': True, 'pr_number': None} + + self._seed_predictions( + mock_validator.mp_storage, uid=0, hotkey='hk_alice', github_id='gh_alice', issue_id=1, preds={1: 0.8} + ) + + _run(merge_predictions(mock_validator, {})) + + assert mock_validator.mp_storage.is_issue_settled(1) is True + + @patch('gittensor.validator.merge_predictions.settlement.GITTENSOR_VALIDATOR_PAT', '') + def test_no_validator_pat_skips(self, mock_validator): + from gittensor.validator.merge_predictions.settlement import merge_predictions + + _run(merge_predictions(mock_validator, {})) + + assert mock_validator.mp_storage.get_all_emas() == [] diff --git a/tests/validator/test_dynamic_open_pr_threshold.py b/tests/validator/test_dynamic_open_pr_threshold.py index f59500bf..c5e63d18 100644 --- a/tests/validator/test_dynamic_open_pr_threshold.py +++ b/tests/validator/test_dynamic_open_pr_threshold.py @@ -14,11 +14,11 @@ EXCESSIVE_PR_PENALTY_BASE_THRESHOLD, MAX_OPEN_PR_THRESHOLD, ) -from gittensor.validator.configurations.tier_config import Tier, TierStats -from gittensor.validator.evaluation.scoring import ( +from gittensor.validator.oss_contributions.scoring import ( calculate_open_pr_threshold, calculate_pr_spam_penalty_multiplier, ) +from gittensor.validator.oss_contributions.tier_config import Tier, TierStats def make_tier_stats( diff --git a/tests/validator/test_emission_shares.py b/tests/validator/test_emission_shares.py new file mode 100644 index 00000000..d81b3445 --- /dev/null +++ b/tests/validator/test_emission_shares.py @@ -0,0 +1,25 @@ +# Entrius 2025 + +""" +Guard-rail test: emission shares must never exceed 100% cumulatively. + +If ISSUES_TREASURY_EMISSION_SHARE + PREDICTIONS_EMISSIONS_SHARE >= 1.0, +OSS contributions would receive zero or negative share, breaking the reward system. + +Run: + pytest tests/validator/test_emission_shares.py -v +""" + +from gittensor.constants import ISSUES_TREASURY_EMISSION_SHARE, PREDICTIONS_EMISSIONS_SHARE + + +def test_combined_emission_shares_leave_room_for_oss(): + """Issue bounties + merge predictions must not consume all emissions.""" + combined = ISSUES_TREASURY_EMISSION_SHARE + PREDICTIONS_EMISSIONS_SHARE + oss_share = 1.0 - combined + + assert combined < 1.0, ( + f'Combined non-OSS emission shares ({ISSUES_TREASURY_EMISSION_SHARE} + {PREDICTIONS_EMISSIONS_SHARE} ' + f'= {combined}) must be < 1.0, otherwise OSS contributions get nothing' + ) + assert oss_share > 0.0 diff --git a/tests/validator/test_load_weights.py b/tests/validator/test_load_weights.py index ac849508..e59d7b4c 100644 --- a/tests/validator/test_load_weights.py +++ b/tests/validator/test_load_weights.py @@ -10,7 +10,7 @@ import pytest -from gittensor.validator.configurations.tier_config import Tier +from gittensor.validator.oss_contributions.tier_config import Tier from gittensor.validator.utils.load_weights import ( LanguageConfig, RepositoryConfig, diff --git a/tests/validator/test_pioneer_dividend.py b/tests/validator/test_pioneer_dividend.py index 01517258..e7b54754 100644 --- a/tests/validator/test_pioneer_dividend.py +++ b/tests/validator/test_pioneer_dividend.py @@ -15,17 +15,18 @@ PIONEER_DIVIDEND_RATE_2ND, PIONEER_DIVIDEND_RATE_REST, ) -from gittensor.validator.configurations.tier_config import TIERS, Tier -from gittensor.validator.evaluation.scoring import ( +from gittensor.validator.oss_contributions.scoring import ( calculate_pioneer_dividends, finalize_miner_scores, ) +from gittensor.validator.oss_contributions.tier_config import TIERS, Tier from tests.validator.conftest import PRBuilder # ========================================================================== # Fixtures # ========================================================================== + @pytest.fixture def builder(): return PRBuilder() @@ -40,6 +41,7 @@ def bronze(): # TestPioneerEligibility # ========================================================================== + class TestPioneerEligibility: """Tests for PullRequest.is_pioneer_eligible instance method.""" @@ -59,14 +61,18 @@ def test_ineligible_without_merge_timestamp(self, builder, bronze): def test_ineligible_below_token_score_threshold(self, builder, bronze): pr = builder.create( - state=PRState.MERGED, tier=bronze, uid=1, + state=PRState.MERGED, + tier=bronze, + uid=1, token_score=MIN_TOKEN_SCORE_FOR_BASE_SCORE - 1, ) assert not pr.is_pioneer_eligible() def test_eligible_at_exact_token_score_threshold(self, builder, bronze): pr = builder.create( - state=PRState.MERGED, tier=bronze, uid=1, + state=PRState.MERGED, + tier=bronze, + uid=1, token_score=MIN_TOKEN_SCORE_FOR_BASE_SCORE, ) assert pr.is_pioneer_eligible() @@ -76,6 +82,7 @@ def test_eligible_at_exact_token_score_threshold(self, builder, bronze): # TestCalculatePioneerDividends # ========================================================================== + class TestCalculatePioneerDividends: """Tests for calculate_pioneer_dividends function.""" @@ -83,8 +90,13 @@ def test_single_miner_gets_no_dividend(self, builder, bronze): """A lone pioneer with no followers earns zero dividend.""" now = datetime.now(timezone.utc) pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now, earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now, + earned_score=0.0, + collateral_score=0.0, ) pr.base_score = 30.0 evals = {1: MinerEvaluation(uid=1, hotkey='h1', merged_pull_requests=[pr])} @@ -96,12 +108,22 @@ def test_pioneer_earns_dividend_from_follower(self, builder, bronze): """Pioneer earns 30% of first follower's earned_score.""" now = datetime.now(timezone.utc) pioneer_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now - timedelta(days=5), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now - timedelta(days=5), + earned_score=0.0, + collateral_score=0.0, ) follower_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=2, - merged_at=now, earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=2, + merged_at=now, + earned_score=0.0, + collateral_score=0.0, ) pioneer_pr.base_score = 30.0 follower_pr.base_score = 20.0 @@ -124,16 +146,26 @@ def test_dividend_from_multiple_followers(self, builder, bronze): """Pioneer dividend uses per-position rates: 30%, 20%, 10%, 10%.""" now = datetime.now(timezone.utc) pioneer_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now - timedelta(days=10), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now - timedelta(days=10), + earned_score=0.0, + collateral_score=0.0, ) pioneer_pr.base_score = 30.0 pioneer_pr.earned_score = 30.0 follower_prs = [] for uid in range(2, 6): # 4 followers pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=uid, - merged_at=now - timedelta(days=10 - uid), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=uid, + merged_at=now - timedelta(days=10 - uid), + earned_score=0.0, + collateral_score=0.0, ) pr.base_score = 10.0 pr.earned_score = 10.0 @@ -148,7 +180,8 @@ def test_dividend_from_multiple_followers(self, builder, bronze): 10.0 * PIONEER_DIVIDEND_RATE_1ST + 10.0 * PIONEER_DIVIDEND_RATE_2ND + 10.0 * PIONEER_DIVIDEND_RATE_REST - + 10.0 * PIONEER_DIVIDEND_RATE_REST, 2 + + 10.0 * PIONEER_DIVIDEND_RATE_REST, + 2, ) assert pioneer_pr.pioneer_dividend == expected_dividend @@ -156,8 +189,13 @@ def test_dividend_grows_with_many_followers(self, builder, bronze): """Dividend scales with followers but is capped at PIONEER_DIVIDEND_MAX_RATIO × own earned.""" now = datetime.now(timezone.utc) pioneer_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now - timedelta(days=30), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now - timedelta(days=30), + earned_score=0.0, + collateral_score=0.0, ) pioneer_pr.base_score = 30.0 pioneer_pr.earned_score = 30.0 @@ -165,8 +203,13 @@ def test_dividend_grows_with_many_followers(self, builder, bronze): follower_prs = [] for uid in range(2, 12): # 10 followers pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=uid, - merged_at=now - timedelta(days=30 - uid), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=uid, + merged_at=now - timedelta(days=30 - uid), + earned_score=0.0, + collateral_score=0.0, ) pr.base_score = 30.0 pr.earned_score = 30.0 @@ -186,15 +229,25 @@ def test_dividend_cap_at_max_ratio(self, builder, bronze): """Dividend is capped at PIONEER_DIVIDEND_MAX_RATIO × pioneer's own earned_score.""" now = datetime.now(timezone.utc) pioneer_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now - timedelta(days=10), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now - timedelta(days=10), + earned_score=0.0, + collateral_score=0.0, ) pioneer_pr.base_score = 10.0 pioneer_pr.earned_score = 10.0 # 1 follower with much higher earned_score follower_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=2, - merged_at=now, earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=2, + merged_at=now, + earned_score=0.0, + collateral_score=0.0, ) follower_pr.base_score = 100.0 follower_pr.earned_score = 100.0 @@ -212,23 +265,43 @@ def test_multiple_follower_prs_summed(self, builder, bronze): """A follower with multiple PRs on the same repo contributes all earned_scores to dividend.""" now = datetime.now(timezone.utc) pioneer_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now - timedelta(days=10), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now - timedelta(days=10), + earned_score=0.0, + collateral_score=0.0, ) pioneer_pr.base_score = 30.0 pioneer_pr.earned_score = 30.0 # Follower has 3 PRs on the same repo f_pr1 = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=2, - merged_at=now - timedelta(days=5), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=2, + merged_at=now - timedelta(days=5), + earned_score=0.0, + collateral_score=0.0, ) f_pr2 = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=2, - merged_at=now - timedelta(days=3), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=2, + merged_at=now - timedelta(days=3), + earned_score=0.0, + collateral_score=0.0, ) f_pr3 = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=2, - merged_at=now - timedelta(days=1), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=2, + merged_at=now - timedelta(days=1), + earned_score=0.0, + collateral_score=0.0, ) f_pr1.base_score = 5.0 f_pr1.earned_score = 5.0 @@ -251,20 +324,40 @@ def test_repos_are_independent(self, builder, bronze): now = datetime.now(timezone.utc) # UID 1 pioneers repo-a, UID 2 pioneers repo-b pr1a = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now - timedelta(days=10), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now - timedelta(days=10), + earned_score=0.0, + collateral_score=0.0, ) pr2a = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=2, - merged_at=now - timedelta(days=5), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=2, + merged_at=now - timedelta(days=5), + earned_score=0.0, + collateral_score=0.0, ) pr2b = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-b', uid=2, - merged_at=now - timedelta(days=10), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-b', + uid=2, + merged_at=now - timedelta(days=10), + earned_score=0.0, + collateral_score=0.0, ) pr1b = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-b', uid=1, - merged_at=now - timedelta(days=5), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-b', + uid=1, + merged_at=now - timedelta(days=5), + earned_score=0.0, + collateral_score=0.0, ) for pr in [pr1a, pr2a, pr2b, pr1b]: pr.base_score = 30.0 @@ -286,15 +379,24 @@ def test_low_quality_pr_excluded_from_pioneer(self, builder, bronze): """Low token_score PR cannot be pioneer; quality follower becomes pioneer.""" now = datetime.now(timezone.utc) snipe_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, merged_at=now - timedelta(days=10), token_score=MIN_TOKEN_SCORE_FOR_BASE_SCORE - 1, - earned_score=0.0, collateral_score=0.0, + earned_score=0.0, + collateral_score=0.0, ) snipe_pr.base_score = 5.0 good_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=2, - merged_at=now - timedelta(days=5), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=2, + merged_at=now - timedelta(days=5), + earned_score=0.0, + collateral_score=0.0, ) good_pr.base_score = 30.0 evals = { @@ -314,16 +416,25 @@ def test_ineligible_pr_does_not_receive_rank(self, builder, bronze): """Ineligible PR from same miner on same repo must not get pioneer_rank.""" now = datetime.now(timezone.utc) eligible_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now - timedelta(days=10), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now - timedelta(days=10), + earned_score=0.0, + collateral_score=0.0, ) eligible_pr.base_score = 30.0 eligible_pr.earned_score = 30.0 ineligible_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, merged_at=now - timedelta(days=5), token_score=MIN_TOKEN_SCORE_FOR_BASE_SCORE - 1, - earned_score=0.0, collateral_score=0.0, + earned_score=0.0, + collateral_score=0.0, ) ineligible_pr.base_score = 2.0 ineligible_pr.earned_score = 2.0 @@ -339,12 +450,24 @@ def test_deterministic_tiebreak_by_pr_number(self, builder, bronze): """Same merged_at timestamp: lower PR number wins pioneer status.""" now = datetime.now(timezone.utc) pr1 = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now, number=10, earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now, + number=10, + earned_score=0.0, + collateral_score=0.0, ) pr2 = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=2, - merged_at=now, number=20, earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=2, + merged_at=now, + number=20, + earned_score=0.0, + collateral_score=0.0, ) pr1.base_score = 30.0 pr2.base_score = 30.0 @@ -361,16 +484,31 @@ def test_only_pioneering_pr_gets_dividend(self, builder, bronze): """Follow-up PRs by the pioneer on same repo don't get dividend.""" now = datetime.now(timezone.utc) pioneer_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now - timedelta(days=10), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now - timedelta(days=10), + earned_score=0.0, + collateral_score=0.0, ) followup_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now - timedelta(days=2), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now - timedelta(days=2), + earned_score=0.0, + collateral_score=0.0, ) follower_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=2, - merged_at=now, earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=2, + merged_at=now, + earned_score=0.0, + collateral_score=0.0, ) pioneer_pr.base_score = 30.0 pioneer_pr.earned_score = 30.0 @@ -397,8 +535,14 @@ def test_no_eligible_prs(self, builder, bronze): """No crash when all PRs are ineligible.""" now = datetime.now(timezone.utc) pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now, token_score=0.0, earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now, + token_score=0.0, + earned_score=0.0, + collateral_score=0.0, ) evals = {1: MinerEvaluation(uid=1, hotkey='h1', merged_pull_requests=[pr])} calculate_pioneer_dividends(evals) @@ -410,6 +554,7 @@ def test_no_eligible_prs(self, builder, bronze): # TestFinalizeWithDividend # ========================================================================== + class TestFinalizeWithDividend: """Integration tests: pioneer dividend flows through finalize_miner_scores.""" @@ -417,12 +562,22 @@ def test_pioneer_dividend_additive_to_earned_score(self, builder, bronze): """Pioneer dividend is added on top of earned_score: base × multipliers + dividend.""" now = datetime.now(timezone.utc) pioneer_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now - timedelta(days=5), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now - timedelta(days=5), + earned_score=0.0, + collateral_score=0.0, ) follower_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=2, - merged_at=now, earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=2, + merged_at=now, + earned_score=0.0, + collateral_score=0.0, ) pioneer_pr.base_score = 30.0 follower_pr.base_score = 30.0 @@ -451,8 +606,13 @@ def test_follower_keeps_full_score(self, builder, bronze): now = datetime.now(timezone.utc) # Create a solo miner scenario for baseline solo_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/solo-repo', uid=3, - merged_at=now, earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/solo-repo', + uid=3, + merged_at=now, + earned_score=0.0, + collateral_score=0.0, ) solo_pr.base_score = 30.0 solo_eval = MinerEvaluation(uid=3, hotkey='h3', merged_pull_requests=[solo_pr]) @@ -460,12 +620,22 @@ def test_follower_keeps_full_score(self, builder, bronze): # Create a follower scenario pioneer_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now - timedelta(days=5), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now - timedelta(days=5), + earned_score=0.0, + collateral_score=0.0, ) follower_pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=2, - merged_at=now, earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=2, + merged_at=now, + earned_score=0.0, + collateral_score=0.0, ) pioneer_pr.base_score = 30.0 follower_pr.base_score = 30.0 @@ -484,6 +654,7 @@ def test_follower_keeps_full_score(self, builder, bronze): # TestPioneerIncentiveEvidence # ========================================================================== + class TestPioneerIncentiveEvidence: """Evidence tests proving the mechanism rewards exploration over pile-on.""" @@ -496,23 +667,27 @@ def test_exploration_beats_pile_on(self, builder, bronze): pile_evals = {} for uid in range(1, 6): pr = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/saturated', uid=uid, - merged_at=now - timedelta(days=uid), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/saturated', + uid=uid, + merged_at=now - timedelta(days=uid), + earned_score=0.0, + collateral_score=0.0, ) pr.base_score = 30.0 pr.earned_score = 30.0 pile_evals[uid] = MinerEvaluation(uid=uid, hotkey=f'h{uid}', merged_pull_requests=[pr]) calculate_pioneer_dividends(pile_evals) - pile_total_dividend = sum( - pr.pioneer_dividend for ev in pile_evals.values() for pr in ev.merged_pull_requests - ) + pile_total_dividend = sum(pr.pioneer_dividend for ev in pile_evals.values() for pr in ev.merged_pull_requests) # With pile-on, only pioneer gets dividend (based on follower earned_scores) expected = round( 30.0 * PIONEER_DIVIDEND_RATE_1ST + 30.0 * PIONEER_DIVIDEND_RATE_2ND + 30.0 * PIONEER_DIVIDEND_RATE_REST - + 30.0 * PIONEER_DIVIDEND_RATE_REST, 2 + + 30.0 * PIONEER_DIVIDEND_RATE_REST, + 2, ) assert pile_total_dividend == expected @@ -523,14 +698,24 @@ def test_pioneer_earns_more_with_more_followers(self, builder, bronze): # Scenario 1: 1 follower builder.reset() pr1 = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=1, - merged_at=now - timedelta(days=10), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=1, + merged_at=now - timedelta(days=10), + earned_score=0.0, + collateral_score=0.0, ) pr1.base_score = 30.0 pr1.earned_score = 30.0 f1 = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-a', uid=2, - merged_at=now, earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-a', + uid=2, + merged_at=now, + earned_score=0.0, + collateral_score=0.0, ) f1.base_score = 30.0 f1.earned_score = 30.0 @@ -544,16 +729,26 @@ def test_pioneer_earns_more_with_more_followers(self, builder, bronze): # Scenario 2: 5 followers builder.reset() pr2 = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-b', uid=1, - merged_at=now - timedelta(days=10), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-b', + uid=1, + merged_at=now - timedelta(days=10), + earned_score=0.0, + collateral_score=0.0, ) pr2.base_score = 30.0 pr2.earned_score = 30.0 followers = [] for uid in range(2, 7): f = builder.create( - state=PRState.MERGED, tier=bronze, repo='org/repo-b', uid=uid, - merged_at=now - timedelta(days=10 - uid), earned_score=0.0, collateral_score=0.0, + state=PRState.MERGED, + tier=bronze, + repo='org/repo-b', + uid=uid, + merged_at=now - timedelta(days=10 - uid), + earned_score=0.0, + collateral_score=0.0, ) f.base_score = 30.0 f.earned_score = 30.0 diff --git a/tests/validator/test_tier_credibility.py b/tests/validator/test_tier_credibility.py index e4c7205d..441ac396 100644 --- a/tests/validator/test_tier_credibility.py +++ b/tests/validator/test_tier_credibility.py @@ -16,7 +16,12 @@ import pytest from gittensor.classes import PRState -from gittensor.validator.configurations.tier_config import ( +from gittensor.validator.oss_contributions.credibility import ( + calculate_credibility_per_tier, + calculate_tier_stats, + is_tier_unlocked, +) +from gittensor.validator.oss_contributions.tier_config import ( TIERS, TIERS_ORDER, Tier, @@ -25,11 +30,6 @@ get_next_tier, get_tier_from_config, ) -from gittensor.validator.evaluation.credibility import ( - calculate_credibility_per_tier, - calculate_tier_stats, - is_tier_unlocked, -) class TestGetNextTier: diff --git a/tests/validator/test_tier_emissions.py b/tests/validator/test_tier_emissions.py index 1ef97489..8ad0f399 100644 --- a/tests/validator/test_tier_emissions.py +++ b/tests/validator/test_tier_emissions.py @@ -15,8 +15,7 @@ from gittensor.classes import MinerEvaluation from gittensor.constants import TIER_EMISSION_SPLITS -from gittensor.validator.configurations.tier_config import Tier, TierStats -from gittensor.validator.evaluation.tier_emissions import allocate_emissions_by_tier +from gittensor.validator.oss_contributions.tier_config import Tier, TierStats, allocate_emissions_by_tier class TestTierEmissionSplitsConstant: diff --git a/tests/validator/test_tier_requirements.py b/tests/validator/test_tier_requirements.py index 6fcd5c68..86495d4b 100644 --- a/tests/validator/test_tier_requirements.py +++ b/tests/validator/test_tier_requirements.py @@ -22,15 +22,15 @@ import pytest -from gittensor.validator.configurations.tier_config import ( - TIERS, - Tier, -) -from gittensor.validator.evaluation.credibility import ( +from gittensor.validator.oss_contributions.credibility import ( calculate_credibility_per_tier, calculate_tier_stats, is_tier_unlocked, ) +from gittensor.validator.oss_contributions.tier_config import ( + TIERS, + Tier, +) class TestCredibilityThresholdBehavior: