diff --git a/validator/app/src/compute_horde_validator/validator/pylon.py b/validator/app/src/compute_horde_validator/validator/pylon.py index dd2c0ce5f..70f6884ab 100644 --- a/validator/app/src/compute_horde_validator/validator/pylon.py +++ b/validator/app/src/compute_horde_validator/validator/pylon.py @@ -1,3 +1,5 @@ +import ipaddress + from django.conf import settings from pylon_client import Hotkey, Neuron, PylonClient @@ -24,5 +26,7 @@ def get_serving_hotkeys(neurons: list[Neuron]) -> list[Hotkey]: list[Hotkey]: List of Hotkey objects. """ return [ - neuron.hotkey for neuron in neurons if neuron.axon_info and neuron.axon_info.ip != "0.0.0.0" + neuron.hotkey + for neuron in neurons + if neuron.axon_info and neuron.axon_info.ip != ipaddress.IPv4Address("0.0.0.0") ] diff --git a/validator/app/src/compute_horde_validator/validator/routing/default.py b/validator/app/src/compute_horde_validator/validator/routing/default.py index cbfb3c11f..da5bd3c8a 100644 --- a/validator/app/src/compute_horde_validator/validator/routing/default.py +++ b/validator/app/src/compute_horde_validator/validator/routing/default.py @@ -3,7 +3,6 @@ from datetime import timedelta from typing import assert_never -from compute_horde.blockchain.block_cache import aget_current_block from compute_horde.executor_class import EXECUTOR_CLASS from compute_horde.fv_protocol.facilitator_requests import ( OrganicJobRequest, @@ -20,11 +19,11 @@ from compute_horde_validator.validator.dynamic_config import aget_config from compute_horde_validator.validator.models import ( ComputeTimeAllowance, - MetagraphSnapshot, Miner, MinerManifest, MinerPreliminaryReservation, ) +from compute_horde_validator.validator.pylon import pylon_client from compute_horde_validator.validator.routing.base import RoutingBase from compute_horde_validator.validator.routing.types import ( AllMinersBusy, @@ -109,13 +108,8 @@ async def _pick_miner_for_job_v2(request: V2JobRequest) -> JobRoute: request.download_time_limit + request.execution_time_limit + request.upload_time_limit ) - block: int | None = None - try: - block = (await MetagraphSnapshot.aget_latest()).block - except Exception as exc: - logger.warning(f"Failed to get latest metagraph snapshot: {exc}") - block = await aget_current_block() - assert block is not None, "Failed to get current block from cache or subtensor." + block = pylon_client().get_latest_block() + assert block is not None, "Failed to get current block from pylon" if not await aget_config("DYNAMIC_ALLOW_CROSS_CYCLE_ORGANIC_JOBS"): seconds_remaining_in_cycle = _get_seconds_remaining_in_current_cycle(block) diff --git a/validator/app/src/compute_horde_validator/validator/routing/tests/conftest.py b/validator/app/src/compute_horde_validator/validator/routing/tests/conftest.py index 6cd7d1479..d6f91ad27 100644 --- a/validator/app/src/compute_horde_validator/validator/routing/tests/conftest.py +++ b/validator/app/src/compute_horde_validator/validator/routing/tests/conftest.py @@ -7,6 +7,8 @@ from compute_horde_validator.validator.organic_jobs.miner_driver import execute_organic_job_request +from ...tests.helpers import mocked_pylon_client + logger = logging.getLogger(__name__) @@ -85,3 +87,13 @@ def miner_keypair(): return bittensor_wallet.Keypair.create_from_mnemonic( "almost fatigue race slim picnic mass better clog deal solve already champion" ) + + +@pytest.fixture +def patch_pylon_client(): + mock_pylon_client = mocked_pylon_client() + with patch( + "compute_horde_validator.validator.routing.default.pylon_client", + return_value=mock_pylon_client, + ): + yield mock_pylon_client diff --git a/validator/app/src/compute_horde_validator/validator/routing/tests/test_job_routing.py b/validator/app/src/compute_horde_validator/validator/routing/tests/test_job_routing.py index 0f6fcabb0..9705ef61e 100644 --- a/validator/app/src/compute_horde_validator/validator/routing/tests/test_job_routing.py +++ b/validator/app/src/compute_horde_validator/validator/routing/tests/test_job_routing.py @@ -18,7 +18,6 @@ from compute_horde_validator.validator.models import ( ComputeTimeAllowance, Cycle, - MetagraphSnapshot, Miner, MinerBlacklist, MinerManifest, @@ -33,6 +32,8 @@ ) from compute_horde_validator.validator.utils import TRUSTED_MINER_FAKE_KEY +from ...tests.helpers import mock_pylon_neuron + JOB_REQUEST = V2JobRequest( uuid=str(uuid.uuid4()), executor_class=DEFAULT_EXECUTOR_CLASS, @@ -64,7 +65,7 @@ def validator(settings): @pytest.fixture(autouse=True) -def setup_db(miners, validator): +def setup_db(miners, validator, patch_pylon_client): # noqa now = timezone.now() cycle = Cycle.objects.create(start=1, stop=2) batch = SyntheticJobBatch.objects.create(block=1, created_at=now, cycle=cycle) @@ -77,16 +78,15 @@ def setup_db(miners, validator): executor_count=5, online_executor_count=5, ) - MetagraphSnapshot.objects.create( - id=MetagraphSnapshot.SnapshotType.LATEST, - block=1, - alpha_stake=[2000] + [0] * len(miners), - tao_stake=[2000] + [0] * len(miners), - stake=[2000] + [0] * len(miners), - uids=list(range(len(miners) + 1)), - hotkeys=[validator.hotkey] + [m.hotkey for m in miners], - serving_hotkeys=["miner_0", "miner_1"], - ) + + neurons = [mock_pylon_neuron(0, validator.hotkey, 2000)] # validator + for i, miner in enumerate(miners): + neurons.append(mock_pylon_neuron(i + 1, miner.hotkey, 0, i < 2)) + + metagraph_data = {"block": 1, "block_hash": "0x123", "neurons": {n.hotkey: n for n in neurons}} + patch_pylon_client.override("get_metagraph", metagraph_data) + patch_pylon_client.override("get_latest_block", 1) + for miner in miners: ComputeTimeAllowance.objects.create( cycle=cycle, @@ -353,17 +353,6 @@ async def test_pick_miner_for_job__collateral_tiebreak_on_equal_allowance_percen assert job_route.miner.hotkey_ss58 == miner2.hotkey -@pytest.mark.django_db(transaction=True) -@pytest.mark.asyncio -@patch("compute_horde_validator.validator.routing.default.aget_current_block") -async def test_pick_miner_for_job__use_subtensor_for_block_on_cache_miss( - mocked_aget_current_block, miners -): - mocked_aget_current_block.return_value = 1 - await MetagraphSnapshot.objects.all().adelete() - await routing().pick_miner_for_job_request(JOB_REQUEST) - - @pytest.mark.django_db(transaction=True) @pytest.mark.override_config(DYNAMIC_ALLOW_CROSS_CYCLE_ORGANIC_JOBS=True) @patch("compute_horde_validator.validator.routing.default._get_seconds_remaining_in_current_cycle") diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index 417634cae..675f495e0 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -39,6 +39,7 @@ from django.utils.timezone import now from numpy.typing import NDArray from pydantic import JsonValue, TypeAdapter +from pylon_common.models import Metagraph from compute_horde_validator.celery import app from compute_horde_validator.validator import collateral @@ -63,7 +64,7 @@ drive_organic_job, execute_organic_job_request, ) -from compute_horde_validator.validator.pylon import pylon_client +from compute_horde_validator.validator.pylon import get_serving_hotkeys, pylon_client from compute_horde_validator.validator.routing.types import JobRoute from compute_horde_validator.validator.s3 import ( download_prompts_from_s3_url, @@ -1271,17 +1272,23 @@ async def get_manifests_from_miners( @app.task def set_compute_time_allowances() -> None: - metagraph = MetagraphSnapshot.get_cycle_start() - current_cycle = Cycle.from_block(metagraph.block, netuid=settings.BITTENSOR_NETUID) + # Get current block to determine the cycle + current_block = pylon_client().get_latest_block() + current_cycle = Cycle.from_block(current_block, netuid=settings.BITTENSOR_NETUID) if current_cycle.set_compute_time_allowance: logger.debug(f"allowances already calculated for cycle {current_cycle}") return + # Get metagraph at cycle start block for consistent allowance calculation + cycle_start_metagraph = pylon_client().get_metagraph(block=current_cycle.start) + set_success = False try: current_cycle.set_compute_time_allowance = True current_cycle.save() - set_success = async_to_sync(_set_compute_time_allowances)(metagraph, current_cycle) + set_success = async_to_sync(_set_compute_time_allowances)( + cycle_start_metagraph, current_cycle + ) except Exception: msg = "Failed to set compute time allowances: {e}" @@ -1298,7 +1305,7 @@ def set_compute_time_allowances() -> None: current_cycle.save() -async def _set_compute_time_allowances(metagraph: MetagraphSnapshot, cycle: Cycle) -> bool: +async def _set_compute_time_allowances(metagraph: Metagraph, cycle: Cycle) -> bool: """ Calculate and save allowances for all validator-miner pairs at the beginning of a cycle. """ @@ -1308,14 +1315,15 @@ async def _set_compute_time_allowances(metagraph: MetagraphSnapshot, cycle: Cycl "cycle_stop": cycle.stop, } - miners_hotkeys = metagraph.get_serving_hotkeys() + metagraph = pylon_client().get_metagraph() + miners_hotkeys = get_serving_hotkeys(metagraph.get_neurons()) if len(miners_hotkeys) == 0: msg = "No miners in the last metagraph snapshot - will NOT set compute time allowances" await save_compute_time_allowance_event(SystemEvent.EventSubType.GIVING_UP, msg, data) logger.warning(msg) return False - total_stake = metagraph.get_total_validator_stake() + total_stake = sum(neuron.stake for neuron in metagraph.get_neurons()) if total_stake is None or total_stake <= 0.0: msg = "Total stake for last_metagraph snapshot is 0 - skipping compute time allowances" await save_compute_time_allowance_event(SystemEvent.EventSubType.GIVING_UP, msg, data) @@ -1325,10 +1333,10 @@ async def _set_compute_time_allowances(metagraph: MetagraphSnapshot, cycle: Cycl # compute stake proportion for each validator hotkey_to_stake_proportion = {} validators_hotkeys = [] - for hotkey, stake in zip(metagraph.hotkeys, metagraph.stake): - if stake > MIN_VALIDATOR_STAKE: - validators_hotkeys.append(hotkey) - hotkey_to_stake_proportion[hotkey] = stake / total_stake + for neuron in metagraph.get_neurons(): + if neuron.stake >= MIN_VALIDATOR_STAKE: + validators_hotkeys.append(neuron.hotkey) + hotkey_to_stake_proportion[neuron.hotkey] = neuron.stake / total_stake logger.debug(f"Validator stake proportion: {hotkey_to_stake_proportion}") data["validator_stake_proportion"] = hotkey_to_stake_proportion diff --git a/validator/app/src/compute_horde_validator/validator/tests/conftest.py b/validator/app/src/compute_horde_validator/validator/tests/conftest.py index 4a95d9dec..af7443ccc 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/conftest.py +++ b/validator/app/src/compute_horde_validator/validator/tests/conftest.py @@ -12,7 +12,7 @@ from pytest_mock import MockerFixture from ..organic_jobs.miner_driver import execute_organic_job_request -from .helpers import MockNeuron, MockSyntheticMinerClient +from .helpers import MockNeuron, MockSyntheticMinerClient, mocked_pylon_client logger = logging.getLogger(__name__) @@ -151,6 +151,15 @@ def run_uuid(): return str(uuid.uuid4()) +@pytest.fixture() +def patch_pylon_client(): + mock_pylon_client = mocked_pylon_client() + with patch( + "compute_horde_validator.validator.tasks.pylon_client", return_value=mock_pylon_client + ): + yield mock_pylon_client + + # NOTE: Use this fixture when you need to find dangling asyncio tasks. It is currently commented # because redis channels layers keeps dangling tasks, that makes the tests fail -_- # @pytest_asyncio.fixture(autouse=True) diff --git a/validator/app/src/compute_horde_validator/validator/tests/helpers.py b/validator/app/src/compute_horde_validator/validator/tests/helpers.py index bc273fcbf..9fb7c1f95 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/helpers.py +++ b/validator/app/src/compute_horde_validator/validator/tests/helpers.py @@ -1,4 +1,5 @@ import glob +import ipaddress import logging import numbers import os @@ -28,6 +29,7 @@ from constance.base import Config from django.conf import settings from pydantic import TypeAdapter +from pylon_client import AxonInfo, Neuron, PylonClient from compute_horde_validator.validator.models import SystemEvent from compute_horde_validator.validator.organic_jobs.miner_client import MinerClient @@ -39,6 +41,36 @@ logger = logging.getLogger(__name__) +def mock_pylon_neuron(uid: int, hotkey: str, stake: float, serving: bool = True) -> Neuron: + return Neuron( + uid=uid, + coldkey=f"coldkey_{uid}", + hotkey=hotkey, + active=True, + axon_info=AxonInfo( + ip=ipaddress.IPv4Address("127.0.0.1" if serving else "0.0.0.0"), + port=8080, + protocol=4, + ), + stake=stake, + rank=0.1, + emission=0.01, + incentive=0.1, + consensus=0.9, + trust=0.9, + validator_trust=0.9, + dividends=0.01, + last_update=12340, + validator_permit=("validator" in hotkey), + pruning_score=0, + ) + + +def mocked_pylon_client(): + mock_data_path = os.path.join(os.path.dirname(__file__), "pylon_mock_data.json") + return PylonClient(mock_data_path=mock_data_path) + + def throw_error(*args): raise Exception("Error thrown for testing") diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_compute_time_allowance.py b/validator/app/src/compute_horde_validator/validator/tests/test_compute_time_allowance.py index 2b74f5e45..e06a9a54b 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_compute_time_allowance.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_compute_time_allowance.py @@ -1,14 +1,14 @@ import logging from datetime import timedelta -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, patch import pytest from compute_horde_core.executor_class import ExecutorClass +from pylon_common.models import AxonInfo, Metagraph from compute_horde_validator.validator.models import ( ComputeTimeAllowance, Cycle, - MetagraphSnapshot, Miner, MinerManifest, SyntheticJobBatch, @@ -18,7 +18,7 @@ _set_compute_time_allowances, set_compute_time_allowances, ) -from compute_horde_validator.validator.tests.helpers import patch_constance +from compute_horde_validator.validator.tests.helpers import mock_pylon_neuron, patch_constance logger = logging.getLogger(__name__) @@ -46,22 +46,19 @@ async def mock_get_manifests_from_miners(miners: list[Miner], timeout: int = 5) def setup_db(): cycle = Cycle.objects.create(start=708, stop=1430) - metagraph = MagicMock(spec=MetagraphSnapshot) - metagraph.block = 720 + metagraph_data = {"block": 720, "block_hash": "0x123", "neurons": {}} + metagraph = Metagraph(**metagraph_data) return cycle, metagraph @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) -def test_set_allowances_skip(): +def test_set_allowances_skip(patch_pylon_client): cycle, metagraph = setup_db() cycle.set_compute_time_allowance = True cycle.save() - with patch( - "compute_horde_validator.validator.tasks.MetagraphSnapshot.get_cycle_start", - return_value=metagraph, - ): - set_compute_time_allowances() + patch_pylon_client.override("get_metagraph", metagraph) + set_compute_time_allowances() cycle.refresh_from_db() assert cycle.set_compute_time_allowance is True @@ -73,18 +70,15 @@ def test_set_allowances_skip(): mock_get_manifests_from_miners, ) @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) -def test_set_allowances_success(): +def test_set_allowances_success(patch_pylon_client): cycle, metagraph = setup_db() with ( - patch( - "compute_horde_validator.validator.tasks.MetagraphSnapshot.get_cycle_start", - return_value=metagraph, - ), patch( "compute_horde_validator.validator.tasks._set_compute_time_allowances", AsyncMock(return_value=True), ), ): + patch_pylon_client.override("get_metagraph", metagraph) set_compute_time_allowances() cycle.refresh_from_db() @@ -92,18 +86,15 @@ def test_set_allowances_success(): @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) -def test_set_allowances_failure(): +def test_set_allowances_failure(patch_pylon_client): cycle, metagraph = setup_db() with ( - patch( - "compute_horde_validator.validator.tasks.MetagraphSnapshot.get_cycle_start", - return_value=metagraph, - ), patch( "compute_horde_validator.validator.tasks._set_compute_time_allowances", AsyncMock(return_value=False), ), ): + patch_pylon_client.override("get_metagraph", metagraph) set_compute_time_allowances() cycle.refresh_from_db() @@ -112,18 +103,15 @@ def test_set_allowances_failure(): @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) -def test_set_allowances_exception(): +def test_set_allowances_exception(patch_pylon_client): cycle, metagraph = setup_db() with ( - patch( - "compute_horde_validator.validator.tasks.MetagraphSnapshot.get_cycle_start", - return_value=metagraph, - ), patch( "compute_horde_validator.validator.tasks._set_compute_time_allowances", side_effect=Exception("Test exception"), ), ): + patch_pylon_client.override("get_metagraph", metagraph) set_compute_time_allowances() cycle.refresh_from_db() @@ -164,14 +152,19 @@ async def asetup_db(num_miners: int = 3, num_validators: int = 2): await MinerManifest.objects.abulk_create(manifest_objects) - metagraph = MagicMock(spec=MetagraphSnapshot) - metagraph.block = 720 - metagraph.get_total_validator_stake.return_value = 1000.0 - metagraph.get_serving_hotkeys.return_value = miner_hotkeys - metagraph.hotkeys = miner_hotkeys + validator_hotkeys - metagraph.stake = [0.0] * len(miner_hotkeys) + [ - (i + 1) * 10 for i in range(len(validator_hotkeys)) - ] + # Create proper Metagraph with neurons + neurons = {} + + # Create miner neurons (with valid axon_info for serving) + for i, hotkey in enumerate(miner_hotkeys): + neurons[hotkey] = mock_pylon_neuron(i, hotkey, 800.0, True) + + # Create validator neurons (high stake, no axon_info) + for i, hotkey in enumerate(validator_hotkeys): + neurons[hotkey] = mock_pylon_neuron(i + len(miner_hotkeys), hotkey, (i + 1) * 1000, False) + + metagraph_data = {"block": 720, "block_hash": "0x123", "neurons": neurons} + metagraph = Metagraph(**metagraph_data) return cycle, metagraph @@ -179,21 +172,28 @@ async def asetup_db(num_miners: int = 3, num_validators: int = 2): "compute_horde_validator.validator.tasks.get_manifests_from_miners", mock_get_manifests_from_miners, ) +@patch("compute_horde_validator.validator.tasks.MIN_VALIDATOR_STAKE", 1000.0) @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) @pytest.mark.asyncio -async def test_set_compute_time_allowances_success(): +async def test_set_compute_time_allowances_success(patch_pylon_client): num_miners, num_validators = 5, 3 cycle, metagraph = await asetup_db(num_miners, num_validators) - logger.info(f"Cycle: {cycle}, Metagraph: {metagraph.hotkeys}") + logger.info( + f"Cycle: {cycle}, Metagraph block: {metagraph.block}, neurons: {len(metagraph.neurons)}" + ) + patch_pylon_client.override("get_metagraph", metagraph) is_set = await _set_compute_time_allowances(metagraph, cycle) assert is_set is True - allowances = [cta async for cta in ComputeTimeAllowance.objects.order_by("initial_allowance")] - assert len(allowances) == num_miners * num_validators - assert allowances[0].initial_allowance == 432.0 - assert allowances[1].initial_allowance == 864.0 - assert allowances[2].initial_allowance == 1296.0 + allowance_values = [ + cta.initial_allowance + async for cta in ComputeTimeAllowance.objects.order_by("initial_allowance") + ] + assert len(allowance_values) == num_miners * num_validators + assert allowance_values[0] == 4320.0 + assert allowance_values[1] == 8640.0 + assert allowance_values[2] == 12960.0 assert await SystemEvent.objects.acount() == 1 system_event = await SystemEvent.objects.aget() @@ -204,9 +204,9 @@ async def test_set_compute_time_allowances_success(): "cycle_stop": cycle.stop, "total_allowance_records_created": num_miners * num_validators, "validator_stake_proportion": { - "validator0": 0.01, - "validator1": 0.02, - "validator2": 0.03, + "validator0": 0.1, + "validator1": 0.2, + "validator2": 0.3, }, } @@ -217,10 +217,32 @@ async def test_set_compute_time_allowances_success(): ) @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) @pytest.mark.asyncio -async def test_set_compute_time_allowances_failure_no_serving_hotkeys(): - cycle, metagraph = await asetup_db() - metagraph.get_serving_hotkeys.return_value = [] +async def test_set_compute_time_allowances_failure_no_serving_hotkeys(patch_pylon_client): + cycle, _ = await asetup_db() + # Create metagraph with no serving miners (all have 0.0.0.0 IPs) + neurons = { + "validator0": { + "uid": 0, + "hotkey": "validator0", + "coldkey": "coldkey_val_0", + "stake": 10, + "axon_info": AxonInfo(ip="0.0.0.0", port=0, protocol=4), + "active": True, + "rank": 0.1, + "emission": 0.01, + "incentive": 0.1, + "consensus": 0.9, + "trust": 0.9, + "validator_trust": 0.9, + "dividends": 0.01, + "last_update": 12340, + "validator_permit": True, + "pruning_score": 0, + } + } + metagraph = Metagraph(block=720, block_hash="0x123", neurons=neurons) + patch_pylon_client.override("get_metagraph", metagraph) is_set = await _set_compute_time_allowances(metagraph, cycle) assert is_set is False assert await ComputeTimeAllowance.objects.acount() == 0 @@ -235,9 +257,10 @@ async def test_set_compute_time_allowances_failure_no_serving_hotkeys(): ) @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) @pytest.mark.asyncio -async def test_set_compute_time_allowances_failure_no_manifests(): +async def test_set_compute_time_allowances_failure_no_manifests(patch_pylon_client): cycle, metagraph = await asetup_db(num_miners=0) + patch_pylon_client.override("get_metagraph", metagraph) is_set = await _set_compute_time_allowances(metagraph, cycle) assert is_set is False assert await ComputeTimeAllowance.objects.acount() == 0 diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_facilitator_client.py b/validator/app/src/compute_horde_validator/validator/tests/test_facilitator_client.py index 499e0d8d5..75c34b874 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_facilitator_client.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_facilitator_client.py @@ -23,6 +23,7 @@ from django.conf import settings from django.utils import timezone +from compute_horde_validator.validator.allowance.types import Miner as AllowanceMiner from compute_horde_validator.validator.models import ( ComputeTimeAllowance, Cycle, @@ -36,6 +37,7 @@ from compute_horde_validator.validator.organic_jobs.facilitator_client import ( FacilitatorClient, ) +from compute_horde_validator.validator.routing.types import JobRoute from compute_horde_validator.validator.utils import MACHINE_SPEC_CHANNEL, TRUSTED_MINER_FAKE_KEY from .helpers import ( @@ -51,12 +53,27 @@ @asynccontextmanager async def async_patch_all(): + async def mock_routing_pick_miner(request): + # TODO: simplify mocking? + return JobRoute( + miner=AllowanceMiner( + address="fakehost", + ip_version=4, + port=1234, + hotkey_ss58=TRUSTED_MINER_FAKE_KEY, + ), + allowance_reservation_id=None, + ) + with ( patch( "compute_horde_validator.validator.organic_jobs.facilitator_client.verify_request_or_fail", return_value=True, ), - patch("turbobt.Bittensor"), + patch( + "compute_horde_validator.validator.routing.default._pick_miner_for_job_v2", + side_effect=mock_routing_pick_miner, + ), ): yield @@ -70,9 +87,16 @@ async def setup_db(n: int = 1): created_at=now, ) miners = [ - await Miner.objects.acreate(hotkey=f"miner_{i}", collateral_wei=Decimal(10**18)) + await Miner.objects.acreate(hotkey=f"hotkey_{i}", collateral_wei=Decimal(10**18)) for i in range(0, n) ] + # Update first miner to be the trusted miner for facilitator tests + miners[0].hotkey = TRUSTED_MINER_FAKE_KEY + miners[0].address = "fakehost" + miners[0].ip_version = 4 + miners[0].port = 1234 + await miners[0].asave() + validator = await Miner.objects.acreate(hotkey=settings.BITTENSOR_WALLET().hotkey.ss58_address) for i, miner in enumerate(miners): await MinerManifest.objects.acreate( @@ -262,6 +286,10 @@ async def serve(self, ws): self.condition.notify() +@pytest.mark.override_config( + DYNAMIC_CHECK_ALLOWANCE_WHILE_ROUTING=False, + DYNAMIC_MINIMUM_COLLATERAL_AMOUNT_WEI=0, +) @pytest.mark.asyncio @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) @pytest.mark.parametrize( @@ -418,6 +446,10 @@ async def test_wait_for_specs(specs_msg: dict): await reap_tasks(task) +@pytest.mark.override_config( + DYNAMIC_CHECK_ALLOWANCE_WHILE_ROUTING=False, + DYNAMIC_MINIMUM_COLLATERAL_AMOUNT_WEI=0, +) @pytest.mark.asyncio @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) @patch( diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_organic_jobs/conftest.py b/validator/app/src/compute_horde_validator/validator/tests/test_organic_jobs/conftest.py index 3b8979acb..250991d7e 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_organic_jobs/conftest.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_organic_jobs/conftest.py @@ -15,7 +15,6 @@ from compute_horde_validator.validator.models import ( ComputeTimeAllowance, Cycle, - MetagraphSnapshot, Miner, MinerManifest, SyntheticJobBatch, @@ -23,6 +22,8 @@ from compute_horde_validator.validator.organic_jobs.facilitator_client import FacilitatorClient from compute_horde_validator.validator.tests.transport import SimulationTransport +from ..helpers import mocked_pylon_client + @pytest.fixture def miner(miner_keypair): @@ -62,20 +63,17 @@ def compute_time_allowance(cycle, miner, validator): ) -# NOTE: Currently this is here to make sure job routing can read current block. -# Other fields are not used now. @pytest.fixture(autouse=True) -def metagraph_snapshot(cycle): - return MetagraphSnapshot.objects.create( - id=MetagraphSnapshot.SnapshotType.LATEST, - block=cycle.start, - alpha_stake=[], - tao_stake=[], - stake=[], - uids=[], - hotkeys=[], - serving_hotkeys=[], - ) +def patch_pylon_client(cycle): + mock_pylon_client = mocked_pylon_client() + mock_pylon_client.override("get_latest_block", cycle.start) + metagraph_data = {"block": cycle.start, "block_hash": "0x123", "neurons": {}} + mock_pylon_client.override("get_metagraph", metagraph_data) + with patch( + "compute_horde_validator.validator.routing.default.pylon_client", + return_value=mock_pylon_client, + ): + yield mock_pylon_client @pytest.fixture(autouse=True) diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_set_scores.py b/validator/app/src/compute_horde_validator/validator/tests/test_set_scores.py index a28b6ca3d..812af2d75 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_set_scores.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_set_scores.py @@ -1,14 +1,11 @@ import concurrent.futures -import os import uuid -from unittest.mock import patch import pytest from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS from constance.test.pytest import override_config from django.test import override_settings from django.utils.timezone import now -from pylon_client import PylonClient from compute_horde_validator.validator.models import ( Cycle, @@ -30,13 +27,6 @@ ) -@pytest.fixture -def mock_pylon_client(): - """Create a mock PylonClient with built-in mocking capabilities.""" - mock_data_path = os.path.join(os.path.dirname(__file__), "pylon_mock_data.json") - return PylonClient(mock_data_path=mock_data_path) - - @pytest.fixture(autouse=True) def _default_commit_reveal_params(): with ( @@ -100,24 +90,18 @@ def test_normalize_scores(): @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) -def test_set_scores__no_batches_found(settings, bittensor, mock_pylon_client): - with patch( - "compute_horde_validator.validator.tasks.pylon_client", return_value=mock_pylon_client - ): - mock_pylon_client.override("get_latest_block", 361) - set_scores() - assert SystemEvent.objects.using(settings.DEFAULT_DB_ALIAS).count() == 0 +def test_set_scores__no_batches_found(settings, bittensor, patch_pylon_client): + patch_pylon_client.override("get_latest_block", 361) + set_scores() + assert SystemEvent.objects.using(settings.DEFAULT_DB_ALIAS).count() == 0 @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) -def test_set_scores__too_early(settings, bittensor, mock_pylon_client): - with patch( - "compute_horde_validator.validator.tasks.pylon_client", return_value=mock_pylon_client - ): - mock_pylon_client.override("get_latest_block", 359) - setup_db() - set_scores() - assert SystemEvent.objects.using(settings.DEFAULT_DB_ALIAS).count() == 0 +def test_set_scores__too_early(settings, bittensor, patch_pylon_client): + patch_pylon_client.override("get_latest_block", 359) + setup_db() + set_scores() + assert SystemEvent.objects.using(settings.DEFAULT_DB_ALIAS).count() == 0 @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) @@ -257,7 +241,7 @@ def test_set_scores__too_early(settings, bittensor, mock_pylon_client): def test_set_scores__weight_commit_success( settings, bittensor, - mock_pylon_client, + patch_pylon_client, cycle_number, burn_rate, burn_partition, @@ -267,97 +251,85 @@ def test_set_scores__weight_commit_success( ): current_block = 1084 + cycle_number * 722 - with patch( - "compute_horde_validator.validator.tasks.pylon_client", return_value=mock_pylon_client + patch_pylon_client.override("get_latest_block", current_block) + + setup_db(cycle_number=cycle_number, hotkey_to_score=hotkey_to_score) + with override_config( + DYNAMIC_BURN_TARGET_SS58ADDRESSES=burn_targets, + DYNAMIC_BURN_RATE=burn_rate, + DYNAMIC_BURN_PARTITION=burn_partition, ): - mock_pylon_client.override("get_latest_block", current_block) - - setup_db(cycle_number=cycle_number, hotkey_to_score=hotkey_to_score) - with override_config( - DYNAMIC_BURN_TARGET_SS58ADDRESSES=burn_targets, - DYNAMIC_BURN_RATE=burn_rate, - DYNAMIC_BURN_PARTITION=burn_partition, - ): - set_scores() - - # Verify pylon client was called with correct weights - mock_pylon_client.mock.set_weights.assert_called_once() - actual_weights = mock_pylon_client.mock.set_weights.call_args.kwargs.get("weights", {}) - - expected_weights_set = expected_weights_committed - for uid, w in expected_weights_set.items(): - actual_w = actual_weights.get(uid) - if actual_w is not None and ((w - actual_w) / w < 0.001): - expected_weights_set[uid] = actual_w - - # Verify database state - Weights object should still be created - from_db = Weights.objects.get() - assert from_db.block == current_block - - assert SystemEvent.objects.using(settings.DEFAULT_DB_ALIAS).count() == 2 - check_system_events( - SystemEvent.EventType.WEIGHT_SETTING_SUCCESS, - SystemEvent.EventSubType.COMMIT_WEIGHTS_SUCCESS, - 1, - ) + set_scores() + + # Verify pylon client was called with correct weights + patch_pylon_client.mock.set_weights.assert_called_once() + actual_weights = patch_pylon_client.mock.set_weights.call_args.kwargs.get("weights", {}) + + expected_weights_set = expected_weights_committed + for uid, w in expected_weights_set.items(): + actual_w = actual_weights.get(uid) + if actual_w is not None and ((w - actual_w) / w < 0.001): + expected_weights_set[uid] = actual_w + + # Verify database state - Weights object should still be created + from_db = Weights.objects.get() + assert from_db.block == current_block + + assert SystemEvent.objects.using(settings.DEFAULT_DB_ALIAS).count() == 2 + check_system_events( + SystemEvent.EventType.WEIGHT_SETTING_SUCCESS, + SystemEvent.EventSubType.COMMIT_WEIGHTS_SUCCESS, + 1, + ) @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) -def test_set_scores__weight_commit_failure(settings, bittensor, mock_pylon_client): +def test_set_scores__weight_commit_failure(settings, bittensor, patch_pylon_client): """Test that pylon client failures are properly handled.""" - with patch( - "compute_horde_validator.validator.tasks.pylon_client", return_value=mock_pylon_client - ): - mock_pylon_client.override("get_latest_block", 1084) - mock_pylon_client.mock.set_weights.side_effect = Exception("Internal Server Error") - setup_db() - set_scores() + patch_pylon_client.override("get_latest_block", 1084) + patch_pylon_client.mock.set_weights.side_effect = Exception("Internal Server Error") + setup_db() + set_scores() - # Verify pylon client was called - mock_pylon_client.mock.set_weights.assert_called_once() + # Verify pylon client was called + patch_pylon_client.mock.set_weights.assert_called_once() - assert SystemEvent.objects.using(settings.DEFAULT_DB_ALIAS).count() == 2 - check_system_events( - SystemEvent.EventType.WEIGHT_SETTING_FAILURE, - SystemEvent.EventSubType.COMMIT_WEIGHTS_ERROR, - 1, - ) + assert SystemEvent.objects.using(settings.DEFAULT_DB_ALIAS).count() == 2 + check_system_events( + SystemEvent.EventType.WEIGHT_SETTING_FAILURE, + SystemEvent.EventSubType.COMMIT_WEIGHTS_ERROR, + 1, + ) @pytest.mark.parametrize("current_block", [723, 999, 1082, 1430, 1443]) @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) def test_set_scores__commit__too_early_or_too_late( - bittensor, current_block: int, mock_pylon_client + bittensor, current_block: int, patch_pylon_client ): - with patch( - "compute_horde_validator.validator.tasks.pylon_client", return_value=mock_pylon_client - ): - mock_pylon_client.override("get_latest_block", current_block) + patch_pylon_client.override("get_latest_block", current_block) - setup_db() - set_scores() + setup_db() + set_scores() - mock_pylon_client.mock.set_weights.assert_not_called() + patch_pylon_client.mock.set_weights.assert_not_called() - assert not Weights.objects.exists() - assert not SystemEvent.objects.exists() + assert not Weights.objects.exists() + assert not SystemEvent.objects.exists() @pytest.mark.django_db(databases=["default", "default_alias"], transaction=True) -def test_set_scores__multiple_starts(settings, bittensor, mock_pylon_client): +def test_set_scores__multiple_starts(settings, bittensor, patch_pylon_client): # to ensure the other tasks will be run at the same time settings.CELERY_TASK_ALWAYS_EAGER = False threads = 5 - with patch( - "compute_horde_validator.validator.tasks.pylon_client", return_value=mock_pylon_client - ): - setup_db() + setup_db() - with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as pool: - for _ in range(threads): - pool.submit(set_scores) - mock_pylon_client.mock.set_weights.assert_called_once() + with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as pool: + for _ in range(threads): + pool.submit(set_scores) + patch_pylon_client.mock.set_weights.assert_called_once() assert SystemEvent.objects.using(settings.DEFAULT_DB_ALIAS).count() == 2