{"object":"hivemind.opentrainer.status","component":"OpenTrainer","version":"1.0.0-s36","status":"TRAINERS_LIVE","status_basis":"1 node heartbeat(s) within 300 s of 1793699223.889","updated_utc":1793699223.889,"portal_category":{"role":"trainer","display_name":"Trainer (Intense GPU Training — OpenTrainer)","description":"Provides dedicated or nocturnal GPU compute for intense distributed training (Stage-2 Tiny-MoE NF4 QLoRA, DiLoCo 2D-DCT outer steps, signed AMCP/Kafka Aglet rounds, and BOINC workunits) via OpenTrainer with zero-port-forwarding firewall traversal.","registered_trainer_accounts_count":0,"registered_trainer_accounts":[]},"live_opentrainer_fleet":{"active_trainer_nodes":1,"distinct_physical_trainer_gpus":1,"total_registered_nodes":6,"nodes":[{"account_id":"ac50df15c6589f99c5150e81c47e5e9e","node_id":"cybervalkyrie-fr-5ecb:XAVGAMSRV-RTX2070","handle":"cybervalkyrie-fr-5ecb","gpu_name":"NVIDIA GeForce RTX 2070","vram_total_mib":8192,"vram_free_mib":6195,"network_mode":"outbound-https-relay","stacks":["qlora-moe","diloco","amcp-aglet","boinc"],"host_label":"XAVGAMSRV-RTX2070","last_heartbeat_utc":1793699194.5433974},{"account_id":"58fd3df19d2b5ef8c10031e1389aac6a","node_id":"rtx-2070-producer-9351:XAVGAMSRV-RTX2070","handle":"rtx-2070-producer-9351","gpu_name":"NVIDIA GeForce RTX 2070","vram_total_mib":8192,"vram_free_mib":6195,"network_mode":"outbound-https-relay","stacks":["qlora-moe","diloco","amcp-aglet","boinc"],"host_label":"XAVGAMSRV-RTX2070","last_heartbeat_utc":1793656282.0759237},{"account_id":"c6ff2861ca5ef572553468dc7ba21671","node_id":"minder_kipping_rtx4080:KIPPING-EXOMOON-RTX4080","handle":"minder_kipping_rtx4080","gpu_name":"NVIDIA GeForce RTX 4080 (16 GB NF4 QLoRA)","vram_total_mib":16384,"vram_free_mib":13200,"network_mode":"outbound-https-relay","stacks":["qlora-moe","diloco","amcp-aglet","boinc","night-federation"],"host_label":"KIPPING-EXOMOON-RTX4080","last_heartbeat_utc":1793653263.8214934},{"account_id":"693133c5529d0c4f18a2fa8a65e96642","node_id":"hm_opentrainer_sentinel:hm-opentrainer-1-cpu-sentinel","handle":"hm_opentrainer_sentinel","gpu_name":"Low-Cost VM CPU Sentinel (e2-small)","vram_total_mib":2048,"vram_free_mib":1792,"network_mode":"outbound-https-relay","stacks":["qlora-moe","diloco","amcp-aglet","boinc","night-federation"],"host_label":"hm-opentrainer-1-cpu-sentinel","last_heartbeat_utc":1793653263.1204197},{"account_id":"ef1f1643bdb0959d3d009d9ea8f53379","node_id":"rtx2070_xavgamsrv_night:XAVGAMSRV-RTX2070","handle":"rtx2070_xavgamsrv_night","gpu_name":"NVIDIA GeForce RTX 2070 (8 GB NF4 QLoRA)","vram_total_mib":8192,"vram_free_mib":6400,"network_mode":"outbound-https-relay","stacks":["qlora-moe","diloco","amcp-aglet","boinc","night-federation"],"host_label":"XAVGAMSRV-RTX2070","last_heartbeat_utc":1793653262.4484692},{"account_id":"f3cfd5b62ba945e9663e4c19fdcded9d","node_id":"xavier_callens:XAVGAMSRV-RTX2070","handle":"xavier_callens","gpu_name":"NVIDIA GeForce RTX 2070","vram_total_mib":8192,"vram_free_mib":6195,"network_mode":"outbound-https-relay","stacks":["qlora-moe","diloco","amcp-aglet","boinc"],"host_label":"XAVGAMSRV-RTX2070","last_heartbeat_utc":1793651379.2857544}]},"moe_self_training_progress":{"object":"hivemind.opentrainer.moe_self_training_progress","target_model":"hivemind/granite-moe-qlora-nf4","architecture":"Stage-2 Tiny Granite-MoE NF4 QLoRA + DiLoCo 2D-DCT Outer Step + Monotone Loss Ratchet","cadence":"every_3_hours_and_continuous_45s_micro_rounds","data_source":"LIVE_RECORDED_OPENTRAINER_ROUNDS","rounds_recorded":30,"latest_round_index":2399,"initial_eval_loss":1.364955,"latest_eval_loss":1.361964,"best_ratcheted_eval_loss":1.361964,"loss_improvement_delta":0.002991,"monotone_ratchet_improved":true,"total_tokens_trained":313344,"active_campaigns":["E8-STAGE2-TINY-GRANITE-MOE-QLORA"],"participating_handles":["cybervalkyrie-fr-5ecb"],"active_trainer_nodes_count":1,"total_registered_trainer_nodes":6},"community_help_guide":[{"topic_id":"producer-openrouter-join","category":"producer_openrouter","title_en":"1. Join as an OpenRouter Inference Producer (Share Idle GPU & Earn HMT)","title_fr":"1. Rejoindre comme Producteur d'Inférence OpenRouter (Partage GPU & HMT)","summary_en":"Run Ollama, llama-server, or vLLM locally and heartbeat your models to the HiveMind OpenRouter gateway. Whenever a local game or heavy workload starts, your GPU yields in < 1.5s.","summary_fr":"Lancez Ollama, llama-server ou vLLM localement et envoyez vos heartbeats vers la passerelle HiveMind OpenRouter. Dès qu'un jeu ou une tâche lourde démarre, votre GPU libère la VRAM en < 1,5 s.","cli_command":"python scripts/hivemind_producer_agent.py --gateway https://hivemind-openrouter-1003063861791.us-central1.run.app --preset coder","verification_endpoint":"/v1/hivemind/producers"},{"topic_id":"trainer-opentrainer-moe","category":"trainer_opentrainer_moe","title_en":"2. Join as an OpenTrainer Self-Training MoE Node (0 Inbound Ports)","title_fr":"2. Contribuer à l'Auto-Entraînement MoE via OpenTrainer (0 Port Entrant)","summary_en":"Participate in Stage-2 Tiny Granite-MoE NF4 QLoRA and DiLoCo 2D-DCT outer steps over outbound HTTPS (TCP/443) or signed AMCP/Kafka Aglets (`hivemind.amcp.federation.v1`). Monotone validation-loss ratchet ensures only loss-improving rounds advance the checkpoint.","summary_fr":"Participez à l'auto-entraînement Stage-2 Tiny Granite-MoE NF4 QLoRA et aux pas externes DiLoCo 2D-DCT via HTTPS sortant (TCP/443) ou Aglets signés AMCP/Kafka (`hivemind.amcp.federation.v1`). Le cliquet monotone rejette toute mise à jour qui dégrade la loss de validation.","cli_command":"python scripts/hivemind_opentrainer_agent.py --gateway https://hivemind-openrouter-1003063861791.us-central1.run.app --network-mode outbound-https-relay --check-firewall --submit-delta --once","verification_endpoint":"/v1/hivemind/opentrainer/status"},{"topic_id":"dual-role-failover-orchestrator","category":"dual_producer_trainer_3h_cycle","title_en":"3. Dual Producer + Trainer Co-Serving & 3-Hour MoE Self-Training Cycle","title_fr":"3. Orchestrateur Hybride Producteur + Entraîneur & Cycle MoE 3 Heures","summary_en":"Every 3 hours, the Producer+Trainer MoE integration orchestrator verifies live OpenRouter inference (`hivemind/coder`, `hivemind/bee-stars` with NIGHT-MOE-1 LoRA, `hivemind/auto`), refreshes producer and OpenTrainer heartbeats, and executes verified QLoRA self-training rounds.","summary_fr":"Toutes les 3 heures, l'orchestrateur d'intégration Producteur+Entraîneur vérifie le service d'inférence OpenRouter (`hivemind/coder`, `hivemind/bee-stars` avec adaptateur NIGHT-MOE-1 LoRA, `hivemind/auto`), rafraîchit les heartbeats Producteur/OpenTrainer et exécute des rondes d'auto-entraînement QLoRA.","cli_command":"PYTHONPATH=src:scripts python scripts/run_3h_producer_trainer_moe_integration.py","verification_endpoint":"/v1/hivemind/contributive-hub"},{"topic_id":"community-troubleshooting-help","category":"community_mutual_aid","title_en":"4. Community Help, Firewall Diagnostics & Honest Adapter Rules","title_fr":"4. Entraide Communautaire, Diagnostic Pare-Feu & Règles d'Intégrité","summary_en":"Behind NAT/CGNAT or Windows Firewall? Use `outbound-https-relay` (TCP/443 outbound only). Note: `hivemind/bee-stars` requires the `NIGHT-MOE-1` LoRA adapter on `llama-server` (`/bee`); plain Ollama nodes should advertise base/coder models without `bee-stars`.","summary_fr":"Derrière un NAT/CGNAT ou pare-feu Windows ? Utilisez `outbound-https-relay` (TCP/443 sortant uniquement). Important : `hivemind/bee-stars` requiert l'adaptateur LoRA `NIGHT-MOE-1` sur `llama-server` (`/bee`) ; un nœud Ollama standard ne doit pas annoncer `bee-stars` sans l'adaptateur.","cli_command":"curl -sS https://hivemind-openrouter-1003063861791.us-central1.run.app/v1/hivemind/contributive-hub","verification_endpoint":"/v1/hivemind/contributive-hub"},{"topic_id":"byom-2-to-4-local-gpus","category":"byom_multi_gpu_co_training","title_en":"5. Co-Train New or Custom Models (BYOM) with 2 to 4 Local GPUs","title_fr":"5. Co-Entraîner de Nouveaux Modèles ou Vos Propres Modèles (BYOM) avec 2 à 4 GPUs","summary_en":"Join as a Trainer (`role=\"trainer\"`) to co-train shared HiveMind MoE models OR your own custom base models/LoRA adapters (`--campaign-id byom-<your-model>`) across 2 to 4 local GPUs (same rig via `CUDA_VISIBLE_DEVICES=0..3` or 2–4 LAN/WAN machines). Each GPU takes a disjoint non-IID data shard (`--learner-index 0..3 --num-learners K`) and merges outer steps via DiLoCo 2D-DCT + Multi-Krum.","summary_fr":"Rejoignez comme Entraîneur (`role=\"trainer\"`) pour co-entraîner de nouveaux modèles communs OU vos propres modèles/adaptateurs LoRA (`--campaign-id byom-<votre-modele>`) avec 2 à 4 GPUs locaux (même machine via `CUDA_VISIBLE_DEVICES=0..3` ou 2 à 4 PC en réseau). Chaque GPU entraîne une partition disjointe (`--learner-index 0..3 --num-learners K`) et synchronise par pas externe DiLoCo 2D-DCT.","cli_command":"for gpu in 0 1 2 3; do CUDA_VISIBLE_DEVICES=$gpu python scripts/hivemind_opentrainer_agent.py --gateway https://hivemind-openrouter-1003063861791.us-central1.run.app --host-label RIG-GPU$gpu --learner-index $gpu --campaign-id byom-custom-moe-v1 --network-mode outbound-https-relay --submit-delta --once & done; wait","verification_endpoint":"/v1/hivemind/opentrainer/status"}],"multi_gpu_2_to_4_byom_playbook":{"title_en":"Co-Training New & Custom (BYOM) Models with 2 to 4 Local GPUs on OpenTrainer","title_fr":"Co-Entraîner de Nouveaux Modèles ou Vos Propres Modèles (BYOM) avec 2 à 4 GPUs Locaux","supported_topologies":[{"topology_id":"single_host_2_to_4_gpus","label_en":"Topology A — 1 Workstation / Homelab Server with 2 to 4 GPUs (CUDA_VISIBLE_DEVICES=0..3)","label_fr":"Topologie A — 1 Station / Serveur Homelab avec 2 à 4 GPUs (CUDA_VISIBLE_DEVICES=0..3)","how_it_works":"Pin one OpenTrainer worker per physical GPU (`CUDA_VISIBLE_DEVICES=0..K-1`), assign each worker a distinct `--host-label RIG-GPU$i` and `--learner-index $i`, and run H=20..40 inner QLoRA steps before outer-step DiLoCo / RDA aggregation."},{"topology_id":"multi_host_lan_or_wan_2_to_4_gpus","label_en":"Topology B — 2 to 4 Separate Gaming PCs / Laptops Across LAN or Internet (0 Inbound Ports)","label_fr":"Topologie B — 2 à 4 PC Gamers / Machines Séparées en LAN ou Internet (0 Port Entrant)","how_it_works":"Each PC runs `scripts/hivemind_opentrainer_agent.py --network-mode outbound-https-relay` (or local syncer `scripts/run_experiment_E8_pilot.py --role syncer --bind 127.0.0.1` over SSH loopback tunnel) and submits only compact FP16/2D-DCT LoRA deltas (~6–18 MB per round, never raw private datasets)."},{"topology_id":"bring_your_own_model_byom","label_en":"Topology C — Bring Your Own Model (BYOM) / Custom Domain LoRA Campaign","label_fr":"Topologie C — Entraîner Votre Propre Modèle (BYOM) / Campagne LoRA Personnalisée","how_it_works":"Specify a custom `--campaign-id byom-<your-project>` when submitting deltas to `/v1/hivemind/opentrainer/submit-delta`, or launch a private 2–4 GPU DiLoCo coordinator locally while optionally publishing the trained GGUF/LoRA bee onto HiveMind OpenRouter (`/v1/chat/completions`)."}],"vram_budget_per_gpu_tier":{"6gb_to_8gb_gpus_e_g_rtx2070_3060_4060":"NF4 4-bit base + rank-4/rank-8 QLoRA (batch=1, grad_accum=8, seq_len=1024, >=1,536 MiB free VRAM safety floor)","12gb_to_16gb_gpus_e_g_rtx3060_12g_4070ti_t4":"NF4/FP8 base + rank-16 QLoRA or 8-expert Tiny-MoE (batch=2, grad_accum=8, seq_len=2048, >=4,300 MiB free VRAM floor on T4)","24gb_plus_gpus_e_g_rtx3090_4090_a5000":"14B–30B MoE expert shards + rank-32 QLoRA / teacher distillation"}},"active_task_offer":{"event_type":"io.hivemind.amcp.fed.task_offer","topic":"hivemind.amcp.federation.v1","run_id":"opentrainer-rtx2070-t4-co-train-v1","round":6499,"algorithm":"e8_qlora_moe_v1","model_id":"hivemind/granite-moe-qlora-nf4","script_sha256":"86cc8db586537fded623058a6b9406f783bd50c824409c8a93da133254c5c44c","base_sha256":"9d29879258386341a4d32b3d60aee5856651dc623cd62211189ad60c49859cdb","global_sha256":"19c78353a7f385e23500b82971463cf6fb4954fa055d4c5037ea346231bc6d4b","inner_steps":40,"inner_lr":0.01,"outer_lr":0.5,"outer_mu":0.5,"qlora_rank":4,"qlora_alpha":8,"quorum":1,"max_delta_l2":25.0,"min_free_vram_mib_t4":4300,"min_free_vram_mib_8gb":1536,"deadline_unix":1793701023.889,"mobile_code_allowed":false,"weights_only_deserialization":true},"recent_training_rounds":[{"submission_id":"ot-7b6f0938ba4e93f5","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2399,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.452973,"eval_loss":1.361964,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793699194.7028553},{"submission_id":"ot-6b09ee08c1920ebc","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2398,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.541976,"eval_loss":1.365776,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793699145.750018},{"submission_id":"ot-a4c0db0a17b8108f","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2397,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.547103,"eval_loss":1.367772,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793699096.3160756},{"submission_id":"ot-5df87ff7b7edad15","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2396,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.56704,"eval_loss":1.373257,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793699046.948073},{"submission_id":"ot-d88d6bcbff1461a3","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2395,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.593824,"eval_loss":1.380023,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698997.1898396},{"submission_id":"ot-7a9b046b4b8c70e6","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2394,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.464317,"eval_loss":1.384537,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698948.356028},{"submission_id":"ot-a016ec33aab14deb","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2393,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.491207,"eval_loss":1.380472,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698899.274502},{"submission_id":"ot-e35afc44179d641a","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2392,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.520765,"eval_loss":1.379757,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698850.3811202},{"submission_id":"ot-401f0cc7f0c1be73","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2391,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.543266,"eval_loss":1.381721,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698800.5537019},{"submission_id":"ot-a35766dacfaa61c5","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2390,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.565765,"eval_loss":1.383401,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698751.427338},{"submission_id":"ot-384b209076dc6e38","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2389,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.582178,"eval_loss":1.385554,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698702.1034281},{"submission_id":"ot-06e3f4a12835e409","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2388,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.511581,"eval_loss":1.391731,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698653.2096586},{"submission_id":"ot-12e6f1eae532a070","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2387,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.540982,"eval_loss":1.398489,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698603.5450852},{"submission_id":"ot-2bb5ebc57028770c","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2386,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.578705,"eval_loss":1.393664,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698554.5311215},{"submission_id":"ot-f886d3765dd98a8a","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2385,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.555041,"eval_loss":1.387283,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698505.2344785},{"submission_id":"ot-0a91ac2ab741e0ca","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2384,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.531933,"eval_loss":1.383613,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698455.9229145},{"submission_id":"ot-19bd06b220b9c6dc","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2383,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.49577,"eval_loss":1.382926,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698406.1671364},{"submission_id":"ot-7a8951b23f8fa5c0","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2382,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.497728,"eval_loss":1.384499,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698357.1132836},{"submission_id":"ot-266bd8ccfae84b41","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2381,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.546371,"eval_loss":1.38329,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698308.0169573},{"submission_id":"ot-d57d692ce458420c","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2380,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.539835,"eval_loss":1.378513,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698259.1512623},{"submission_id":"ot-b3abeb8e6c8337a6","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2379,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.55953,"eval_loss":1.368609,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698209.351405},{"submission_id":"ot-d1629156016baa4b","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2378,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.688713,"eval_loss":1.37279,"tokens_trained":12288,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698183.0957909},{"submission_id":"ot-8fcd404d19e85ba0","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2377,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.667636,"eval_loss":1.381049,"tokens_trained":12288,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698180.0719965},{"submission_id":"ot-2b8c4a41607b5408","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2376,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.719927,"eval_loss":1.388339,"tokens_trained":12288,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698177.2787936},{"submission_id":"ot-870f50989870b9ba","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2375,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.55415,"eval_loss":1.383039,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698160.7960868},{"submission_id":"ot-7667336fd0617754","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2374,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.515054,"eval_loss":1.374669,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698112.0185559},{"submission_id":"ot-0773a35130832d50","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2373,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.616992,"eval_loss":1.373703,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698062.7313578},{"submission_id":"ot-9f29f34c5568107f","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2372,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.586161,"eval_loss":1.371136,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793698013.38286},{"submission_id":"ot-7b1e9270270d7f6e","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2371,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.564567,"eval_loss":1.369921,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793697964.1846673},{"submission_id":"ot-cf76d27ba7daeecb","account_id":"ac50df15c6589f99c5150e81c47e5e9e","handle":"cybervalkyrie-fr-5ecb","run_id":"E8-STAGE2-TINY-GRANITE-MOE-QLORA","round_index":2370,"host_label":"cybervalkyrie-fr-5ecb","network_mode":"outbound-https-relay","delta_l2":3.57,"eval_loss":1.364955,"tokens_trained":10240,"vram_free_mib":6400,"vram_total_mib":8192,"utc":1793697914.7192774}],"firewall_traversal_modes":[{"mode_id":"outbound-https-relay","tier":"Tier B (Default — Zero Firewall / NAT Config)","inbound_ports_required":0,"protocol":"Outbound HTTPS (TCP/443) to /v1/hivemind/opentrainer/*","how_it_works":"The trainer node initiates outbound-only HTTPS requests to OpenTrainer on the Cloud Run gateway. OpenTrainer validates signed Aglet task claims and QLoRA/DiLoCo delta submissions and bridges them into the AMCP/Kafka event ledger. Works behind any home router, carrier-grade NAT (CGNAT), or corporate firewall.","recommended_for":"Home Windows/Linux/macOS GPUs (RTX 2070, RTX 3060, RTX 4070) without public IPs or UDP 51820."},{"mode_id":"amcp-kafka-aglet","tier":"Tier A (Direct AMCP v1.5 + Kafka Signed Task Agents)","inbound_ports_required":0,"protocol":"Outbound TCP/9092 over WireGuard overlay (10.77.0.0/24) to 3 Kafka brokers","how_it_works":"Learners subscribe to topic hivemind.amcp.federation.v1, verify HMAC-signed task_offer envelopes (code_pin SHA-256 with CRLF normalization + base/global tensor SHA-256, weights_only=True, 64 KiB cap), train inner_steps on their non-IID shard, and publish task_result deltas.","recommended_for":"Dedicated nodes on the 10.77.0.0/24 overlay (GCP Tesla T4 + RTX 2070 via WireGuard T4 Hub)."},{"mode_id":"ssh-loopback-tunnel","tier":"Two-Host WAN Cell Tunnel (Loopback-Only Syncer)","inbound_ports_required":0,"protocol":"Outbound SSH port-forward (127.0.0.1:18766 -> 127.0.0.1:8765, permitopen restricted)","how_it_works":"Per PR #401 and PR #407, the syncer never binds 0.0.0.0; it listens strictly on 127.0.0.1:8765 / 10.77.0.4:8765. The remote RTX 2070 learner (XAVGAMSRV) forwards a local loopback port over SSH with a key restricted to permitopen=\"127.0.0.1:8765\" and no shell execution.","recommended_for":"Cross-host E8 Stage 1 (CPU RDA) and Stage 2 (NF4 QLoRA Tiny-MoE) experiments when UDP is blocked."},{"mode_id":"wireguard-t4-hub","tier":"Tier A Overlay Hub & Open-Source Mesh Alternatives","inbound_ports_required":0,"protocol":"Outbound UDP/51820 to 34.48.48.154:51820 (or wstunnel TCP/443, Headscale, Nebula MIT)","how_it_works":"The T4 VM acts as a single WireGuard hub (deploy/amcp/hm-wg-hub.sh) forwarding 10.77.0.5/32 (RTX 2070) to 10.77.0.0/24 (Kafka brokers) with iptables drop rules blocking VPC access. Where UDP/51820 is blocked, trainers use wstunnel (WireGuard over WebSocket/TLS), Headscale + Tailscale DERP, or Nebula certificates.","recommended_for":"Persistent multi-host co-training between XAVGAMSRV (RTX 2070) and socreateai-agora-hermes-node1 (Tesla T4)."},{"mode_id":"boinc-night-workunit","tier":"Tier C (BOINC Nocturnal Workunits & Idle-GPU Worker)","inbound_ports_required":0,"protocol":"Outbound HTTPS (TCP/443) to HiveMind La Ruche BOINC Coordinator","how_it_works":"Standard BOINC client (boinccmd) and hivemind_boinc_idle_worker.py lease nocturnal workunits (22:00-07:00 Paris time), verify that folding ran on the gated CUDA device (PR #408), pause automatically within 4s of gamer input (PR #399), and enforce the monotone validation-loss ratchet before assimilating QLoRA deltas (PR #366).","recommended_for":"Volunteer gamers and citizen-science trainers contributing overnight GPU cycles."}],"aglet_amcp_kafka_lifecycle":[{"aglet_verb":"dispatch","amcp_event_type":"io.hivemind.amcp.fed.task_offer","direction":"OpenTrainer Coordinator -> Trainers (AMCP/Kafka or HTTPS /opentrainer/heartbeat)","security_invariant":"Carries run_id, round, code_pin SHA-256, base_sha256, global_sha256, hyperparameters, deadline, quorum — NEVER executable code."},{"aglet_verb":"arrival (onArrival)","amcp_event_type":"check_offer (local verification gate)","direction":"Trainer Local Verification before GPU execution","security_invariant":"Verifies CRLF-normalized code_pin matches local git checkout, verifies base_sha256 & global_sha256 with weights_only=True, checks night window & free VRAM floor (>= 4,300 MiB on T4 / >= 1,536 MiB on 8 GB RTX 2070)."},{"aglet_verb":"clone","amcp_event_type":"io.hivemind.amcp.fed.task_claim","direction":"Trainer -> OpenTrainer Coordinator","security_invariant":"Multiple independent trainers (e.g. Learner 0 on Tesla T4, Learner 1 on RTX 2070) claim and run the same offer on disjoint non-IID data shards."},{"aglet_verb":"result","amcp_event_type":"io.hivemind.amcp.fed.task_result","direction":"Trainer -> OpenTrainer Coordinator (/v1/hivemind/opentrainer/submit-delta or AMCP bus)","security_invariant":"Sends only the weight delta; coordinator verifies run_id, round, allowed learner list, global_sha256, code_pin, tensor shapes, finiteness (no NaN/Inf) and L2 norm bound before token-weighted outer step."},{"aglet_verb":"retract","amcp_event_type":"io.hivemind.amcp.fed.task_abort","direction":"OpenTrainer Coordinator -> Trainers","security_invariant":"Withdraws and re-offers a round if quorum is not reached by deadline; late work from stale rounds is never carried over (HM-S9-01)."}],"rtx2070_t4_integration_ledger":[{"pr_number":410,"branch":"feat/e8-stage2-tiny-moe-qlora","title":"Signed task agents over AMCP & Kafka (Aglet life cycle, no mobile code), Stage 2 Tiny-MoE QLoRA & live churn soak","components":["src/hivemind/swarm/federation_agents.py","scripts/e8_amcp_round.py","scripts/e8_qlora_moe_round.py","deploy/amcp/hm-wg-hub.sh","docs/FEDERATION_V2_AMCP_AGLETS.md","docs/E8_STAGE2_TINY_MOE_QLORA.md","docs/OVERLAY_OPEN_SOURCE_TUNNELS.md"],"measured_results":{"stage2_qlora_moe_loss":"4.1721 -> 1.7772 (4 rounds, NF4 + LoRA r=4, alpha=8, 0.0000 diff vs single-process baseline)","stage2_negative_control_shuffled_loss":"4.1721 -> 4.2034 (stays at chance ln(64)=4.159)","amcp_kafka_live_cluster_6_rounds":"0.0000 loss difference vs single-process baseline across 3 Spot VM Kafka brokers + T4","amcp_kafka_70_round_churn_soak":"62/70 complete, 8/70 closed by quorum during SIGKILL outages, 0 rejections, loss 4.171 -> 1.448","vram_peak_allocated_mib":27.8,"vram_free_floor_mib":4300}},{"pr_number":409,"branch":"feat/s36-honest-gpu-count-and-producer-autostart","title":"Honest physical GPU counting (#402), cross-platform auto-start service & GPU outreach campaign","components":["src/hivemind/router/accounts.py","src/hivemind/router/t4_visibility.py","scripts/hivemind_producer_autostart.py","scripts/hivemind_gpu_outreach_campaign.py"],"measured_results":{"distinct_physical_gpus_metric":"Normalizes endpoint URLs by _endpoint_host_root; separates distinct_physical_gpus from shared_host_endpoints","autostart_platforms":"Linux (systemd --user), Windows 10/11 (Scheduled Task), macOS (launchd)"}},{"pr_number":408,"branch":"fix/idle-worker-truthful-gpu-probe","title":"Idle GPU worker reports READY only when the fold ran on the gated CUDA device","components":["scripts/hivemind_boinc_idle_worker.py"],"measured_results":{"truthful_cuda_probe":"Refuses READY status if PyTorch/CUDA falls back to CPU or fails the device gate"}},{"pr_number":407,"branch":"artifacts/e8-two-host-wan-results","title":"First real two-host E8 cell between GCP Tesla T4 and Windows 11 RTX 2070 (XAVGAMSRV)","components":["artifacts/experiments/e8_real/RESULTS_T4_SIDE.md"],"measured_results":{"hosts":"Learner 0: socreateai-agora-hermes-node1 (Tesla T4, Linux) | Learner 1: XAVGAMSRV (RTX 2070, Windows 11)","paired_loss_diff_vs_baseline":"+4.8e-07 +/- 8.3e-07 across seeds 1-3 (K=2, 12 rounds, 400 inner steps, RDA merge)","round_latency_s":"p50 = 1.20 s, p95 = 3.79 s, max = 3.89 s (excluding join round 0)"}},{"pr_number":401,"branch":"docs/e8-two-host-real-test","title":"Real two-host test T4 & RTX 2070 per Foundation Paper; pilot syncer refuses 0.0.0.0 wildcard bind","components":["docs/E8_TWO_HOST_REAL_TEST.md","scripts/run_experiment_E8_pilot.py","tests/test_e8_pilot_network_safety.py"],"measured_results":{"network_safety":"Syncer binds 127.0.0.1 by default and raises ValueError on 0.0.0.0 / :: wildcard addresses"}},{"pr_number":399,"branch":"feat/volunteer-gamer-config","title":"Fail-safe gamer idle-GPU helper, stock-BOINC gamer settings & honest join page wording","components":["scripts/hivemind_volunteer_config.py","src/hivemind/router/join_page.py"],"measured_results":{"gamer_preemption":"Unmeasurable input defaults to playing (pauses GPU); generates cc_config.xml + global_prefs_override.xml"}},{"pr_number":366,"branch":"fix/boinc-qlora-ratchet-and-mixed-domain-moe","title":"BOINC QLoRA monotone validation-loss ratchet & mixed-domain MoE dispatch","components":["src/hivemind/boinc/real_client/assimilator.py"],"measured_results":{"monotone_ratchet":"Rejects any outer-step checkpoint where E_val(W^(t+1)) > E_val(W^(t))"}}]}