fork mindXtrain from GitHub (Professor-Codephreak/mindXtrain@661bd41) as the mindX-specific line

.env.example

@@ -0,0 +1,78 @@
+# MI300X non-negotiable runtime environment (mindxtrain2.md §13)
+PYTORCH_ROCM_ARCH=gfx942
+HSA_NO_SCRATCH_RECLAIM=1
+HIP_FORCE_DEV_KERNARG=1
+GPU_MAX_HW_QUEUES=1
+NVTE_CK_USES_BWD_V3=1
+NVTE_CK_IS_V3_ATOMIC_FP32=1
+PRIMUS_TURBO_ATTN_V3_ATOMIC_FP32=1
+NCCL_MIN_NCHANNELS=112
+
+# Operator runtime — inference backend selection
+MINDXTRAIN_BACKEND=vllm
+MINDXTRAIN_VLLM_BASE_URL=http://localhost:8000/v1
+MINDXTRAIN_OPENAI_BASE_URL=https://api.openai.com/v1
+MINDXTRAIN_OPENAI_API_KEY=
+MINDXTRAIN_PERSONA_PATH=/home/hacker/mindX/personas/codephreak.json
+
+# Teacher model used by data/synth.py for synthetic-data rollouts
+MINDXTRAIN_TEACHER_BASE_URL=http://localhost:8000/v1
+MINDXTRAIN_TEACHER_MODEL=Qwen/Qwen3.5-8B
+
+# External storage providers
+HF_TOKEN=
+HF_HUB_USERNAME=
+LIGHTHOUSE_API_KEY=
+LIGHTHOUSE_BASE_URL=https://node.lighthouse.storage
+IPFS_API_URL=http://127.0.0.1:5001
+
+# Provenance / on-chain
+MINDXTRAIN_FACILITATOR_URL=https://facilitator.bankon.io/x402
+MINDXTRAIN_REGISTRY_ADDR=
+MINDXTRAIN_BASE_RPC_URL=https://sepolia.base.org
+MINDXTRAIN_ALGORAND_ALGOD_URL=https://mainnet-api.algonode.cloud
+MINDXTRAIN_ALGORAND_INDEXER_URL=https://mainnet-idx.algonode.cloud
+MINDXTRAIN_BANKON_ENS_URL=https://ens.bankon.pythai.net
+
+# Deploy targets
+MINDXTRAIN_API_BASE_URL=https://mindx.pythai.net
+MINDXTRAIN_AGENTICPLACE_URL=https://agenticplace.pythai.net
+
+# Public /v1/training/jobs bearer-auth secret. Leave blank for an open
+# dev-mode operator; set in production so mindX agents and external callers
+# must present `Authorization: Bearer <this>`. Generate with `openssl rand -hex 32`.
+MINDXTRAIN_API_KEY=
+
+# mindX home — used by `data.source: mindx_dreams` recipes if they don't
+# pin an absolute `data.path`.
+MINDXTRAIN_MINDX_HOME=/home/hacker/mindX
+
+# Local registry (deploy hot-swap)
+MINDXTRAIN_REGISTRY_PATH=./out/registry.json
+
+# Observability (optional)
+MINDXTRAIN_OTEL_ENDPOINT=
+MINDXTRAIN_PROMETHEUS_PORT=9090
+
+# Coach UI deploy — GitHub source-tree push
+GITHUB_TOKEN=
+GITHUB_REPO=professor-codephreak/mindXtrain
+GITHUB_DEFAULT_BRANCH=main
+GITHUB_AUTHOR_NAME=mindXtrain bot
+GITHUB_AUTHOR_EMAIL=noreply@pythai.net
+
+# Coach UI deploy — AMD Dev Cloud MI300X provisioning
+AMD_DEV_CLOUD_TOKEN=
+AMD_DEV_CLOUD_API_BASE=https://api.devcloud.amd.com
+AMD_DEV_CLOUD_SSH_KEY_ID=56216059
+AMD_DEV_CLOUD_REGION=atl1
+AMD_DEV_CLOUD_SIZE=gpu-mi300x8-1536gb-devcloud
+AMD_DEV_CLOUD_IMAGE=vllm-0-17-1
+AMD_DEV_CLOUD_TAGS=mindx,train,aglm,agenticplace,pythai
+
+# Coach UI deploy — existing droplet sync (also used by provisioning to ssh in)
+DROPLET_HOST=
+DROPLET_USER=root
+DROPLET_SSH_KEY=~/.ssh/id_ed25519
+DROPLET_REMOTE_PATH=/workspace/mindxtrain
+DROPLET_CONTAINER=rocm/primus:v26.2

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+docs/blueprints/Winning[[:space:]]the[[:space:]]AMD[[:space:]]x[[:space:]]lablab.ai[[:space:]]Developer[[:space:]]Hackathon[[:space:]]with[[:space:]]mindX[[:space:]]and[[:space:]]xtrain_[[:space:]]A[[:space:]]Three-Track[[:space:]]Strategic[[:space:]]Brief.pdf filter=lfs diff=lfs merge=lfs -text
+docs/blueprints/mindXtrain[[:space:]]Framework_[[:space:]]GLM-5.1,[[:space:]]aGLM[[:space:]]Lineage,[[:space:]]and[[:space:]]Qwen3.5[[:space:]]Primary[[:space:]]Base[[:space:]]Strategy.pdf filter=lfs diff=lfs merge=lfs -text
+docs/blueprints/mindXtrain_[[:space:]]Production[[:space:]]Blueprint[[:space:]]for[[:space:]]the[[:space:]]AMD[[:space:]]and[[:space:]]lablab.ai[[:space:]]Hackathon.pdf filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci.yml

@@ -0,0 +1,68 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  lint-type-test:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "0.8.18"
+
+      - name: Set up Python 3.12
+        run: uv python install 3.12
+
+      - name: Sync workspace (base + ml + data for the runtime checks)
+        run: uv sync --frozen --extra ml --extra data
+
+      - name: Ruff (package + tests)
+        run: uv run ruff check mindxtrain/ tests/
+
+      - name: Mypy (strict-checked paths only)
+        run: uv run mypy mindxtrain/config mindxtrain/provenance
+
+      - name: Pytest
+        run: uv run pytest -q
+
+  build-container:
+    # Build the Containerfile but don't push — proves the recipe still builds
+    # on each PR. Push-to-GHCR happens on tag, in a separate job.
+    runs-on: ubuntu-24.04
+    needs: lint-type-test
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build container (no push)
+        run: docker build -f Containerfile -t mindxtrain:ci .
+
+  publish-container:
+    # On tags vX.Y.Z, push the container to GHCR.
+    runs-on: ubuntu-24.04
+    needs: lint-type-test
+    if: startsWith(github.ref, 'refs/tags/v')
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build + push (tag)
+        run: |
+          TAG="${GITHUB_REF##*/}"
+          IMAGE="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/mindxtrain"
+          docker build -f Containerfile -t "$IMAGE:$TAG" -t "$IMAGE:latest" .
+          docker push "$IMAGE:$TAG"
+          docker push "$IMAGE:latest"

.gitignore

@@ -0,0 +1,34 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+.uv-cache/
+uv.lock.bak
+
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+
+.env
+.env.*
+!.env.example
+creds.api
+*.creds
+secrets/
+
+runs/
+checkpoints/
+out/
+*.safetensors
+*.bin
+*.pt
+*.onnx
+
+.cache/
+autotune_plan.json
+custmodel/runs/
+
+# Agent working dirs (plans, memory, local settings) — never commit.
+.claude/
+
+.DS_Store

AGENTS.md

@@ -0,0 +1,82 @@
+# AGENTS.md
+
+Canonical entry point for any agent (Claude Code, Codex, Cursor, Copilot, etc.) working in this repository. Read this first.
+
+## TL;DR
+
+`mindxtrain` is a single-package Python framework that fine-tunes open-weight LLMs on AMD MI300X and serves them OpenAI-compatible. The differentiator is a **60-second AOT autotune probe** (`mindxtrain bench`) — the plan is fixed at training start; **JIT autotune is forbidden in the production loop**.
+
+The base install is CPU-only and runs the CLI, Coach UI, `bench --dry-run`, manifest verify, and the operator FastAPI. Heavyweight paths gate on opt-in `--extra` groups.
+
+## Where to look
+
+| You want to… | Read |
+|---|---|
+| Understand the architecture and invariants | [`CLAUDE.md`](CLAUDE.md), [`docs/architecture.md`](docs/architecture.md) |
+| Take the repo from "code done" to "demo live" | [`HANDOFF.md`](docs/HANDOFF.md) (11 ordered steps) |
+| Know what's real Python vs. what needs `--extra <group>` | [`docs/actualization_status.md`](docs/actualization_status.md) |
+| Add a recipe / training backend / operator backend / training method | [`docs/development.md`](docs/development.md) §"Adding…" |
+| Look up every CLI verb's options + exit codes | [`docs/cli.md`](docs/cli.md) |
+| Look up every YAML field | [`docs/yaml_schema.md`](docs/yaml_schema.md) |
+| Understand the autotune probe | [`docs/autotune.md`](docs/autotune.md) |
+| Understand the Coach UI | [`docs/coach.md`](docs/coach.md) |
+| Understand classroom / boardroom / dojo governance | [`docs/governance.md`](docs/governance.md) |
+| See the frozen design briefs | [`docs/blueprints/`](docs/blueprints/) |
+
+## Verification gates (must all pass before pushing)
+
+```bash
+uv run ruff check .
+uv run mypy mindxtrain/config mindxtrain/provenance
+uv run pytest -q       # → 564 passed
+```
+
+CI runs the same three commands on Ubuntu 24.04 / Python 3.12 (CPU-only).
+
+## Non-negotiable invariants
+
+These are encoded in the schema/recipes; violating them is a deployment bug. Full reasoning in [`docs/development.md`](docs/development.md).
+
+1. **AOT-only** — no `torch.compile(mode="max-autotune")`, no JIT autotune in vLLM. The YAML field `autotune.policy: aot_only` is the contract.
+2. **`hardware.gpus: Literal[1, 8]`** — 2/4-GPU MI300X FSDP groups hit asymmetric xGMI; schema rejects them at parse time.
+3. **Seven MI300X env vars** are defaults in every recipe's `train.env` (`PYTORCH_ROCM_ARCH=gfx942`, `HSA_NO_SCRATCH_RECLAIM=1`, `HIP_FORCE_DEV_KERNARG=1`, `GPU_MAX_HW_QUEUES=1`, `NVTE_CK_USES_BWD_V3=1`, `NVTE_CK_IS_V3_ATOMIC_FP32=1`, `PRIMUS_TURBO_ATTN_V3_ATOMIC_FP32=1`, `NCCL_MIN_NCHANNELS=112`).
+4. **`extra: forbid` + `frozen: true`** on every Pydantic model.
+5. **Solidity contracts are write-once** — no proxies, no `Ownable`, no admin keys, no setters.
+6. **Numpy pinned `<2.0`** against `torch==2.9.1+rocm7.2.1.lw`.
+7. **Lazy imports for optional deps** — `import mindxtrain.eval.harness` must succeed even without `--extra eval`. Error messages must include the exact `uv sync --extra <group>` to run.
+8. **Reuse boundaries** — Codephreak persona JSON loads at runtime via `MINDXTRAIN_PERSONA_PATH` from `/home/hacker/mindX/`, not by copying bytes. Never reuse `/home/hacker/aglm/` (broken per its own README).
+9. **Clean-room policy** — mindXtrain (and Coach) is a clean-room codebase: functionality from mindX or any external source is **reimplemented/adapted locally from behavior or spec, never copied byte-for-byte**. Consume foreign artifacts (dream corpus, persona) through runtime boundaries; do not vendor foreign code. mindXtrain and Coach **train models** — the dataset/persona/imprint machinery is mindXtrain-native. See [`CLAUDE.md`](CLAUDE.md) §"Clean-room policy".
+
+## Quick commands
+
+```bash
+uv sync                                                       # base, CPU-only
+uv sync --all-extras                                          # everything except amd-quark
+uv run mindxtrain --help                                      # 9 verbs
+uv run mindxtrain init --list                                 # 12 built-in YAML recipes
+uv run mindxtrain bench --dry-run --out plan.json             # synthetic plan, no GPU
+uv run mindxtrain receipt manifest.json --config run.yaml     # BLAKE3 round-trip
+uv run uvicorn mindxtrain.operator.app:app --port 8080        # → /coach/ UI
+```
+
+GPU verbs (`bench` without `--dry-run`, `train`, `quantize`, `serve`) require an AMD MI300X with ROCm 7.2.1 inside `rocm/primus:v26.2`. See [`HANDOFF.md`](docs/HANDOFF.md).
+
+## Available skills
+
+For Algorand-related work (the provenance/x402/ASA paths) the following skills are available and should be preferred over ad-hoc patterns:
+
+- `algorand-typescript`, `build-smart-contracts`, `test-smart-contracts`, `call-smart-contracts` — contract authoring/testing/deploy.
+- `use-algokit-utils`, `use-algokit-cli`, `troubleshoot-errors`, `implement-arc-standards` — client-side AlgoKit work.
+- `search-algorand-examples` — patterns from official Algorand repos.
+- `algorand-ts-migration` — TEALScript / beta → Algorand TypeScript 1.0.
+- `mindx` — for cross-cutting work that touches the broader mindX system.
+
+For the Solidity side (`contracts/`):
+
+- `foundry-framework`, `solidity-dev`, `solidity-style-guide`, `slither-analysis`, `echidna-fuzzer`, `gas-optimization`, `hardhat-framework`.
+
+## When in doubt
+
+- The schema is the source of truth for "what's a valid YAML." Run `uv run python -c "from mindxtrain.config.loader import load_config; load_config('path/to.yaml')"`.
+- The blueprints under `docs/blueprints/` are frozen — do not edit them; they are the historical specification.
+- Per-module status: `docs/actualization_status.md`. If a module says "stub" there, do not silently turn it on; it's stubbed deliberately.

CLAUDE.md

@@ -0,0 +1,118 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What this is
+
+`mindxtrain` is a single-package Python training framework for fine-tuning open-weight LLMs on AMD MI300X and serving them through an OpenAI-compatible API. The architectural differentiator is a **60-second AOT autotune probe** (`mindxtrain bench`) that fixes attention backend (CK vs Triton), GEMM heuristic, and RCCL config at training start — **JIT autotune is forbidden in the production training loop**.
+
+The base install is CPU-only and runs the CLI, Coach UI, `bench --dry-run`, manifest verify, and the operator FastAPI; heavyweight paths gate on opt-in dep groups.
+
+## Commands
+
+The repo uses `uv` with Python 3.12 (pinned `>=3.12,<3.13`). All commands run from the repo root.
+
+```bash
+uv sync                         # base install (CPU-only; 564 tests pass)
+uv sync --extra ml --extra eval --extra data    # opt into heavyweight groups
+uv sync --all-extras            # everything except amd-quark (ships in container)
+
+# Standard local cycle — CI runs the same:
+uv run ruff check .                                       # lint
+uv run mypy mindxtrain/config mindxtrain/provenance       # mypy --strict (only these two)
+uv run pytest -q                                          # → 564 passed
+uv run pytest tests/test_config_schema.py -q              # single test file
+uv run pytest tests/test_config_schema.py::test_xgmi_2gpu_rejected   # single test
+
+# CLI entry point (typer; 9 verbs):
+uv run mindxtrain --help
+uv run mindxtrain init --list                             # list 12 built-in YAML recipes
+uv run mindxtrain init --template qwen3_8b_sft_lora --out run.yaml
+uv run mindxtrain bench --dry-run --out plan.json         # CPU-safe (real probe needs MI300X)
+uv run mindxtrain receipt ./out/runs/<name>/manifest.json --config run.yaml
+
+# Operator FastAPI + Coach UI (no GPU required):
+uv run uvicorn mindxtrain.operator.app:app --host 0.0.0.0 --port 8080
+# → http://localhost:8080/coach/
+```
+
+GPU verbs (`bench` without `--dry-run`, `train`, `quantize`, `serve`) require an AMD MI300X with ROCm 7.2.1 inside `rocm/primus:v26.2`. The full operator path is in `docs/HANDOFF.md`.
+
+Solidity contracts live in `contracts/` (Foundry, solc 0.8.26): `forge install && forge test` from inside `contracts/`.
+
+## Architecture (concentric layers)
+
+The codebase is organized so each inner layer is consumed by the next, never the reverse:
+
+1. **CLI** (`mindxtrain/cli/main.py`, typer) — `init | bench | train | dataset prep | eval | quantize | serve | publish | receipt`. Never reaches into the training backend; consumes a Pydantic-validated config + `AutotunePlan` and dispatches downward.
+2. **Autotune** (`mindxtrain/autotune/`) — the differentiator. Emits `AutotunePlan` JSON, AOT-only.
+3. **Dataset** (`mindxtrain/data/`) — curate → dedupe (MinHash + SemDeDup) → filter → tokenize → pack → synth → verify.
+4. **Training** (`mindxtrain/train/`) — backend dispatch (`dispatch.py` → axolotl / unsloth / torchtune / primus for MI300X subprocess; `trl_cpu` for CPU and `trl_local` for device-aware consumer-GPU/CPU-fallback, both in-process TRL). Methods: SFT, DPO, ORPO, GRPO, GSPO, RLHF, tool-use, CPT.
+5. **Artifact + Integration** (`mindxtrain/{eval,deploy,storage,provenance,operator}`) — Quark FP8/MXFP4 → lm-eval-harness → HF Hub push → Lighthouse pin → mindX register → AgenticPlace → BANKON ENS → x402 metering → ERC-8004 attestation.
+
+Key end-to-end flow: `XTrainConfig` (Pydantic) + `AutotunePlan` → `dispatch_training()` → `checkpoint_dir/` → `eval.json` → `quantized/` → `manifest.json` (BLAKE3 of YAML+dataset+ckpt+eval, plus HF/Lighthouse/INFT/ASA pointers) → operator serves on `/v1/chat/completions`. `mindxtrain receipt` re-hashes and verifies the manifest round-trip.
+
+Recipes live as YAML at `mindxtrain/train/recipes/<name>.yaml` and are auto-picked up by `mindxtrain init --list` and validated by `tests/test_config_schema.py::test_all_recipes_validate`.
+
+## Non-negotiable invariants
+
+These are encoded in the schema/recipes; violating them is a deployment bug, not a style issue.
+
+1. **AOT-only.** No `torch.compile(mode="max-autotune")` in production paths. No JIT autotune in vLLM (`VLLM_USE_TRITON_FLASH_ATTN=0` if needed). `autotune.policy: aot_only` is the YAML contract.
+2. **`hardware.gpus: Literal[1, 8]` only.** 2/4-GPU MI300X FSDP groups hit asymmetric xGMI bandwidth — schema rejects them at parse time. (Tested in `test_config_schema.py::test_xgmi_2gpu_rejected`, `test_distributed.py`.)
+3. **Seven MI300X env vars** are defaults in every recipe's `train.env` (autotune plan may override values, never remove keys): `PYTORCH_ROCM_ARCH=gfx942`, `HSA_NO_SCRATCH_RECLAIM=1`, `HIP_FORCE_DEV_KERNARG=1`, `GPU_MAX_HW_QUEUES=1`, `NVTE_CK_USES_BWD_V3=1`, `NVTE_CK_IS_V3_ATOMIC_FP32=1`, `PRIMUS_TURBO_ATTN_V3_ATOMIC_FP32=1`, `NCCL_MIN_NCHANNELS=112`.
+4. **`extra: forbid` + `frozen: true`** on every Pydantic model — unknown YAML keys raise `ValidationError`; loaded configs are immutable.
+5. **Solidity contracts are write-once.** No proxies, no `Ownable`, no admin keys, no setters in `contracts/src/{mindxtrain_registry,x402_receiver}.sol`. Rotating any parameter requires a fresh deploy.
+6. **Numpy pinned `<2.0`** against `torch==2.9.1+rocm7.2.1.lw`.
+7. **Container is `rocm/primus:v26.2`**; SHA256 digest snapshot in `ops/containerfiles/digest.lock`.
+
+## Lazy-import pattern (mandatory for optional deps)
+
+Optional dep groups: `ml` (trl, transformers, peft, accelerate, datasets), `eval` (lm-eval, lighteval, inspect-ai, jinja2), `data` (datasketch, sentence-transformers, faiss-cpu, pyarrow), `serve` (vllm), `chain` (web3, py-algorand-sdk, huggingface-hub), `obs` (opentelemetry-sdk, prometheus-client, psutil).
+
+Every module that wants an optional dep guards the import inside the function that needs it. `import mindxtrain.eval.harness` must always succeed even without `--extra eval`. Error messages must include the exact `uv sync --extra <group>` to run. New modules taking optional deps must follow this pattern.
+
+## Clean-room policy (non-negotiable)
+
+mindXtrain is a **clean-room** codebase: functionality that originates in another
+project (mindX, external repos, reference implementations) is **reimplemented or
+adapted locally from observed behavior or a spec — never copied byte-for-byte**. The
+in-tree code is owned by mindXtrain and untainted by foreign source.
+
+When you need something from another codebase:
+1. **Load it at runtime** via a documented env var / file path (e.g. the Codephreak
+   persona via `MINDXTRAIN_PERSONA_PATH`), or
+2. **Reimplement the behavior locally** in mindXtrain style, citing the source as a
+   *reference*, not pasting it.
+
+This applies to the whole product, including the Coach: **mindXtrain and Coach train
+models**, and the training/dataset/persona machinery is mindXtrain-native — it consumes
+mindX artifacts (dream corpus, persona) through boundaries, it does not vendor mindX code.
+
+## Reuse boundaries
+
+- **From `/home/hacker/mindX/`** (production codebase): Codephreak persona JSON loaded at runtime via `MINDXTRAIN_PERSONA_PATH`. Do not copy file bytes — load via env var (clean-room).
+- **Not** from `/home/hacker/aglm/` — broken per its own README.
+
+## Adding things
+
+- **New recipe** → drop YAML at `mindxtrain/train/recipes/<name>.yaml`; `test_all_recipes_validate` picks it up.
+- **New training backend** → add `mindxtrain/train/backend_<name>.py` exposing `run_<name>(cfg, plan, out_dir) -> Path`; wire into `train/dispatch.py`; add to `TrainingBackend` literal in `config/schema.py`.
+- **New operator backend** → subclass `Backend` in `mindxtrain/operator/backends/<name>.py` decorated `@register_backend("<name>")`; side-effect import from `models/registry.py`; add runtime branch in `operator/app.py::chat_completions`.
+- **New training method** → add `_MethodBase` subclass in `config/schema.py` with `kind: Literal["<name>"]`; extend `TrainMethod` discriminated union; add `train/<name>.py` runner; update dispatch; add a recipe.
+
+## Documentation hub
+
+| Doc | What it covers |
+|-----|----------------|
+| `docs/NAV.md` | **Docs index** — start here; one line per doc, grouped. |
+| `docs/HANDOFF.md` | 11-step operator checklist (local → MI300X droplet → submission). |
+| `docs/architecture.md` | 5-layer architecture + MI300X invariants + data flow. |
+| `docs/development.md` | Toolchain, optional-deps, lazy-import pattern, debugging table. |
+| `docs/actualization_status.md` | Per-module map of what's real vs. requires extras. |
+| `docs/autotune.md` | The 60-second AOT probe — the differentiator. |
+| `docs/cli.md` | Every verb with synopsis, options, exit codes. |
+| `docs/yaml_schema.md` | Every field of the 10-section `XTrainConfig`. |
+| `docs/coach.md` | Interactive `/coach/` web UI bundled in the operator. |
+| `docs/governance.md` | classroom / boardroom (any-N consensus) / dojo (prime-N dispute settlement). |
+| `docs/blueprints/` | Frozen source design briefs (the spec the project was built against). |

Containerfile

@@ -0,0 +1,25 @@
+# Default-discovered Containerfile for `podman build .` at the repo root.
+# Canonical content also lives at ops/containerfiles/containerfile_train.
+#
+# Pin: rocm/primus:v26.2 ships ROCm 7.2.1, PyTorch 2.9.1, Primus-Turbo with FlashAttention,
+# AITER, and hipBLASLt. SHA256 digest in ops/containerfiles/digest.lock.
+FROM docker.io/rocm/primus:v26.2
+
+ENV PYTORCH_ROCM_ARCH=gfx942 \
+    HSA_NO_SCRATCH_RECLAIM=1 \
+    HIP_FORCE_DEV_KERNARG=1 \
+    GPU_MAX_HW_QUEUES=1 \
+    NVTE_CK_USES_BWD_V3=1 \
+    NVTE_CK_IS_V3_ATOMIC_FP32=1 \
+    PRIMUS_TURBO_ATTN_V3_ATOMIC_FP32=1 \
+    NCCL_MIN_NCHANNELS=112
+
+WORKDIR /workspace/mindxtrain
+
+RUN pip install --no-cache-dir uv
+
+COPY . /workspace/mindxtrain
+RUN uv sync --frozen --no-dev
+
+ENTRYPOINT ["uv", "run", "mindxtrain"]
+CMD ["--help"]

FORK.json

@@ -0,0 +1,12 @@
+{
+ "kind": "source fork (code), mindX-specific line",
+ "upstream_repo": "https://github.com/Professor-Codephreak/mindXtrain",
+ "upstream_commit": "661bd411738d11e633b25c681bbd5676556bab7d",
+ "forked_at_utc": "2026-09-14T20:11:14Z",
+ "licence": "Apache-2.0 (LICENSE, NOTICE and upstream notices carried unchanged)",
+ "excluded": "uncommitted working-tree changes of the upstream checkout (only the committed tree at upstream_commit is here)",
+ "readme_modified": "a fork header and Hugging Face model-card metadata were prepended; the upstream text is unchanged below it",
+ "files": 338,
+ "tree_sha256": "08823c89092f14535575ac8b9493b88912cf19ffa699acc1de35e46848020f68",
+ "proceeds_here": "mindX-specific training work; the GitHub upstream remains agnostic"
+}

LICENSE

@@ -0,0 +1,190 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for describing the origin of the Work and
+      reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Support. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or support.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2026 mindX
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

LICENSE-MIT-upstream-glm51

@@ -0,0 +1,36 @@
+GLM-5.1 — Upstream MIT License notice
+=====================================
+
+This file reproduces, verbatim, the upstream MIT license that applies to the
+GLM-5.1 family of model weights and tokenizer assets distributed by Z.ai
+(formerly Zhipu AI) at https://huggingface.co/zai-org/.
+
+mindxtrain redistributes derived training scaffolding under the Apache
+License 2.0 (see LICENSE), but any model weights or tokenizer files
+downloaded from the upstream Z.ai repository remain subject to the
+upstream license below. Adopters fine-tuning GLM-5.1 must preserve this
+notice in their downstream artifact.
+
+----------------------------------------------------------------------
+
+MIT License
+
+Copyright (c) 2025 Z.ai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

NOTICE

@@ -0,0 +1,58 @@
+mindxtrain
+Copyright (c) 2026 BANKON / mindX
+
+This product includes software developed by the mindX project, distributed
+under the Apache License, Version 2.0 (see LICENSE).
+
+----------------------------------------------------------------------
+
+This product depends on software from the following upstream projects.
+Their copyright and license notices are reproduced below as required by
+their respective licenses.
+
+* Qwen3.5 / Qwen3.6 model weights — Alibaba Cloud
+  License: Apache License 2.0
+  Source: https://huggingface.co/Qwen/
+
+* GLM-5.1 model weights — Z.ai (formerly Zhipu AI)
+  License: MIT License (see LICENSE-MIT-upstream-glm51)
+  Source: https://huggingface.co/zai-org/
+
+* DeepSeek V3.2 model weights — DeepSeek
+  License: DeepSeek License
+  Source: https://huggingface.co/deepseek-ai/
+
+* Mistral Large 3 model weights — Mistral AI
+  License: Mistral Research License / Apache 2.0 (per release)
+  Source: https://huggingface.co/mistralai/
+
+* Phi-4-mini model weights — Microsoft Research
+  License: MIT License
+  Source: https://huggingface.co/microsoft/
+
+* Instella-3B-Instruct model weights — AMD
+  License: Apache License 2.0
+  Source: https://huggingface.co/amd/Instella-3B-Instruct
+
+* AMD ROCm, Primus, AITER, Composable Kernel, hipBLASLt, RCCL — AMD
+  License: MIT / Apache 2.0 (per project)
+  Source: https://github.com/ROCm/
+
+* PyTorch — Linux Foundation / contributors
+  License: BSD 3-Clause
+  Source: https://github.com/pytorch/pytorch
+
+* vLLM — vllm-project
+  License: Apache License 2.0
+  Source: https://github.com/vllm-project/vllm
+
+* TRL, transformers, datasets, accelerate, peft — Hugging Face
+  License: Apache License 2.0
+  Source: https://github.com/huggingface/
+
+* Lighthouse Storage SDK — Lighthouse Web3 Inc.
+  License: MIT
+  Source: https://github.com/lighthouse-web3/
+
+The full text of each upstream license is preserved in their respective
+upstream repositories.

README.md

@@ -0,0 +1,132 @@
+---
+license: apache-2.0
+library_name: mindxtrain
+pipeline_tag: text-generation
+tags:
+- mindx
+- mindxtrain
+- training-framework
+- lora
+- cpu-training
+- proof-of-recall
+datasets:
+- PYTHAI/mindXascension
+- PYTHAI/mindX-docs
+---
+
+> **mindXtrain for mindX — the Hugging Face fork.** This repository is the **mindX-specific** line of mindXtrain,
+> forked on 2026-09-14 from the agnostic upstream
+> [github.com/Professor-Codephreak/mindXtrain](https://github.com/Professor-Codephreak/mindXtrain) at commit
+> [`661bd41`](https://github.com/Professor-Codephreak/mindXtrain/commit/661bd411738d11e633b25c681bbd5676556bab7d) (provenance in [`FORK.json`](FORK.json)).
+> The upstream stays agnostic; mindX-specific training work proceeds **here**:
+>
+> ```bash
+> git clone https://huggingface.co/PYTHAI/mindXtrain
+> ```
+>
+> What this line trains for: the mindX lineage ([`PYTHAI/mindXascension`](https://huggingface.co/datasets/PYTHAI/mindXascension)),
+> built from mindX's doctrine ([`PYTHAI/mindX-docs`](https://huggingface.co/datasets/PYTHAI/mindX-docs), with
+> [the mapping](https://huggingface.co/datasets/PYTHAI/mindX-docs/blob/main/MAPPING.md)); the last accepted generation is
+> [`PYTHAI/mindXtrain39`](https://huggingface.co/PYTHAI/mindXtrain39). The Hub footprint is mapped in
+> [`examples/mindx/HUGGINGFACE_MAP.md`](examples/mindx/HUGGINGFACE_MAP.md). The upstream README follows unchanged.
+
+# mindxtrain
+
+Production training framework for fine-tuning open-weight LLMs on AMD MI300X
+and serving them through an OpenAI-compatible API. Single ordered package,
+canonical layout per [`docs/blueprints/mindXtrain2.md`](docs/blueprints/mindXtrain2.md)
+§Part 4.
+
+The single architectural feature that distinguishes mindxtrain from Axolotl,
+LLaMA-Factory, Unsloth, torchtune and Primus is its **60-second AOT autotune
+probe**: CK-vs-Triton attention, hipBLASLt heuristic, RCCL config — the plan
+is fixed at training start, JIT autotune is forbidden in the production loop.
+
+**Status**: production deployment in progress. The CPU-only base install passes
+its full pytest suite (ruff + mypy clean); with the training extras installed the
+suite is 672 green. Many modules ship as real Python on a CPU-only laptop;
+heavyweight training, eval, and quantization paths gate on opt-in extra dep
+groups. See [`docs/actualization_status.md`](docs/actualization_status.md) for the
+per-module map and [`HANDOFF.md`](docs/HANDOFF.md) for the operator checklist.
+
+## Where this runs
+
+- **Operator + Coach UI:** [https://mindx.pythai.net/coach](https://mindx.pythai.net/coach)
+- **Public training-jobs API:** `https://mindx.pythai.net/v1/training/jobs`
+  (bearer auth via `MINDXTRAIN_API_KEY`)
+- **mindX self-training loop:** mindX's dream cycle writes JSONL training
+  data; this framework consumes it via the `mindx_dreams` data source and
+  fine-tunes a small fallback model on a single MI300X.
+
+## Prove it trains
+
+mindXtrain doesn't just assert that training works — it proves recall. The
+[**dcoach**](docs/dcoach.md) proof loop (`/coach/dcoach`) imprints a persona onto a
+tiny model on CPU, then measures whether the model *recalls* it: the **classroom**
+scores recall before vs after training, the **boardroom** rules success or failure,
+and the verdict feeds an **autotune feedback loop** that tunes the next run. A clean
+CPU run reports a positive imprint Δ (e.g. recall 0.07 → 0.28) and an approved
+verdict. [`docs/NAV.md`](docs/NAV.md) is the full documentation hub.
+
+## Quickstart
+
+```bash
+uv sync                                                    # base install
+uv run pytest -q                                           # → 564 passed
+uv run mindxtrain --help                                   # 9 verbs
+uv run mindxtrain init --template qwen3_8b_sft_lora --out run.yaml
+uv run mindxtrain bench --dry-run --out plan.json
+uv run uvicorn mindxtrain.operator.app:app --host 0.0.0.0 --port 8080
+# open http://localhost:8080/coach/  for the interactive UI
+```
+
+To unlock training / eval / quantize / publish, install the matching dep group:
+
+```bash
+uv sync --extra ml --extra eval --extra data         # train + eval + curate
+# or
+uv sync --all-extras                                  # everything except amd-quark
+```
+
+GPU steps (`bench` without `--dry-run`, `train`, `quantize`, `serve`) require
+an AMD MI300X with ROCm 7.2.1; run inside `rocm/primus:v26.2`. The full
+operator checklist lives in [`HANDOFF.md`](docs/HANDOFF.md).
+
+## Layout
+
+```
+mindxtrain/{cli,config,data,models,train,eval,autotune,
+            operator,storage,provenance,deploy,budget}/   # 99 modules
+contracts/        Foundry workspace for ERC-8004 attestation registry
+ops/              containerfiles, compose, k8s, vmm, gensyn
+tests/            pytest suite — 566 tests, CPU-only smoke
+examples/         demo YAML configs
+docs/             user-facing documentation + frozen blueprints
+scripts/          dev helpers
+```
+
+## Documentation
+
+| Doc | What it covers |
+|-----|----------------|
+| [`HANDOFF.md`](docs/HANDOFF.md) | **Operator checklist** — ordered steps from local setup to live deployment. |
+| [`docs/quickstart.md`](docs/quickstart.md) | Install + base-vs-extras command tour. |
+| [`docs/architecture.md`](docs/architecture.md) | Canonical layout + 5-layer architecture + MI300X invariants. |
+| [`docs/actualization_status.md`](docs/actualization_status.md) | Per-module map of what's real vs. requires extras. |
+| [`docs/autotune.md`](docs/autotune.md) | The 60-second AOT probe — the architectural differentiator. |
+| [`docs/coach.md`](docs/coach.md) | Interactive `/coach/` web UI bundled in the operator. |
+| [`docs/dcoach.md`](docs/dcoach.md) | The dcoach proof loop — prove a CPU model recalls its training; decentralized-training fit. |
+| [`docs/cli.md`](docs/cli.md) | Every `mindxtrain` verb with synopsis, options, exit codes. |
+| [`docs/yaml_schema.md`](docs/yaml_schema.md) | Every field of the 10-section `XTrainConfig`. |
+| [`docs/benchmarks.md`](docs/benchmarks.md) | Target metrics + the 7-cell framework comparison. |
+| [`docs/development.md`](docs/development.md) | Toolchain, optional-deps, lazy-import pattern, invariants. |
+| [`docs/blueprints/`](docs/blueprints/) | Source design briefs (frozen specification). |
+| [`llm.txt`](llm.txt) | Orientation for another model — what is measured, what is not, the traps. |
+| [`examples/mindx/`](examples/mindx/HUGGINGFACE_MAP.md) | Example consumer — mindX on the Hugging Face Hub: its lineage, [docs dataset + mapping](https://huggingface.co/datasets/PYTHAI/mindX-docs/blob/main/MAPPING.md), Spaces and licence-pinned base models. The framework stays agnostic. |
+
+## License
+
+Apache-2.0. See [LICENSE](LICENSE), [NOTICE](NOTICE), and the upstream-license
+notices in [`LICENSE-MIT-upstream-glm51`](LICENSE-MIT-upstream-glm51) and
+[`LICENSE-NOTICE.md`](docs/LICENSE-NOTICE.md). Version history in
+[`CHANGELOG.md`](docs/CHANGELOG.md).

WordPress.agent.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f4b50160d20c34831212970cd5a042523a4c3ff37dc22efaae4c5129de088b1
+size 38690

compose.yaml

@@ -0,0 +1,48 @@
+# Default-discovered compose file for `podman-compose up` at the repo root.
+# Wires the operator FastAPI app -> vLLM-ROCm on the MI300X droplet.
+# Canonical dev stack also lives at ops/compose/compose_dev.yaml.
+
+services:
+  vllm:
+    image: docker.io/rocm/vllm-dev:rocm7.2.1
+    container_name: mindxtrain-vllm
+    ipc: host
+    devices:
+      - /dev/kfd
+      - /dev/dri
+    group_add:
+      - video
+    cap_add:
+      - SYS_PTRACE
+    security_opt:
+      - seccomp=unconfined
+    environment:
+      PYTORCH_ROCM_ARCH: gfx942
+      HSA_NO_SCRATCH_RECLAIM: "1"
+      HIP_FORCE_DEV_KERNARG: "1"
+      GPU_MAX_HW_QUEUES: "1"
+    volumes:
+      - ./out/runs:/workspace/runs:ro
+    command: >
+      vllm serve /workspace/runs/latest/quantized
+      --tensor-parallel-size 1
+      --max-model-len 8192
+      --port 8000
+    ports:
+      - "8000:8000"
+
+  operator:
+    build:
+      context: .
+      dockerfile: Containerfile
+    container_name: mindxtrain-operator
+    depends_on:
+      - vllm
+    environment:
+      MINDXTRAIN_BACKEND: vllm
+      MINDXTRAIN_VLLM_BASE_URL: http://vllm:8000/v1
+      MINDXTRAIN_PERSONA_PATH: /home/hacker/mindX/personas/codephreak.json
+    command: >
+      uvicorn mindxtrain.operator.app:app --host 0.0.0.0 --port 8080
+    ports:
+      - "8080:8080"

contracts/.gitignore

@@ -0,0 +1,4 @@
+out/
+cache/
+broadcast/
+lib/

contracts/README.md

@@ -0,0 +1,32 @@
+# mindXtrain on-chain contracts
+
+Foundry workspace for the immutable, no-admin, no-proxy contracts that anchor
+mindXtrain run receipts and receive x402 settlement proofs.
+
+## Contracts
+
+- `src/mindxtrain_registry.sol` — write-once anchoring of `(yamlHash, datasetCidHash, checkpointCidHash, evalReportHash)` per `runId`.
+- `src/x402_receiver.sol` — records x402 settlement proofs from a fixed facilitator address.
+
+Both follow the cypherpunk2048 standard: no upgradeable proxies, no `Ownable`, no admin keys, no pause function, no setters. Rotating any parameter requires a fresh deployment.
+
+## Day-2 setup (on the MI300X droplet, where Foundry installs alongside the training stack)
+
+```bash
+curl -L https://foundry.paradigm.xyz | bash && foundryup
+cd contracts
+forge install foundry-rs/forge-std --no-git
+forge build
+forge test --gas-report
+```
+
+## Deploy
+
+```bash
+export DEPLOYER_PRIVATE_KEY=0x...
+export X402_FACILITATOR=0x...     # parsec-wallet or Coinbase facilitator
+export BASE_SEPOLIA_RPC_URL=https://sepolia.base.org
+forge script script/Deploy.s.sol --rpc-url base_sepolia --broadcast --verify
+```
+
+Day-3 deploy moves to `--rpc-url base` (mainnet).

contracts/foundry.toml

@@ -0,0 +1,28 @@
+[profile.default]
+src = "src"
+out = "out"
+libs = ["lib"]
+test = "test"
+script = "script"
+solc_version = "0.8.26"
+optimizer = true
+optimizer_runs = 200
+via_ir = false
+fs_permissions = [{ access = "read", path = "./" }]
+
+[fmt]
+line_length = 120
+tab_width = 4
+bracket_spacing = false
+int_types = "long"
+
+[fuzz]
+runs = 256
+
+[rpc_endpoints]
+base        = "${BASE_RPC_URL}"
+base_sepolia = "${BASE_SEPOLIA_RPC_URL}"
+
+[etherscan]
+base         = { key = "${BASESCAN_API_KEY}", chain = "base" }
+base_sepolia = { key = "${BASESCAN_API_KEY}", chain = "base-sepolia" }

contracts/script/Deploy.s.sol

@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: Apache-2.0
+pragma solidity ^0.8.26;
+
+import {Script, console} from "forge-std/Script.sol";
+import {MindXTrainRegistry} from "../src/mindxtrain_registry.sol";
+import {X402Receiver} from "../src/x402_receiver.sol";
+
+contract Deploy is Script {
+    function run() external {
+        uint256 pk = vm.envUint("DEPLOYER_PRIVATE_KEY");
+        address facilitator = vm.envAddress("X402_FACILITATOR");
+        // ("algorand", 203977300) — Algorand mainnet USDC ASA
+        bytes32 assetIdHash = keccak256(abi.encode("algorand", uint256(203977300)));
+
+        vm.startBroadcast(pk);
+        MindXTrainRegistry registry = new MindXTrainRegistry();
+        X402Receiver receiver = new X402Receiver(facilitator, assetIdHash);
+        vm.stopBroadcast();
+
+        console.log("MindXTrainRegistry deployed at:", address(registry));
+        console.log("X402Receiver deployed at:", address(receiver));
+    }
+}

contracts/src/mindxtrain_registry.sol

@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: Apache-2.0
+pragma solidity ^0.8.26;
+
+/// @title MindXTrainRegistry
+/// @notice Write-once anchoring contract for mindXtrain run receipts.
+/// @dev Cypherpunk2048 standard: immutable, no proxy, no Ownable, no admin keys,
+///      no pause, no setter. Receipts cannot be overwritten or revoked.
+contract MindXTrainRegistry {
+    struct Receipt {
+        bytes32 yamlHash;
+        bytes32 datasetCidHash;
+        bytes32 checkpointCidHash;
+        bytes32 evalReportHash;
+        address publisher;
+        uint64 timestamp;
+    }
+
+    mapping(bytes32 => Receipt) private _receipts;
+
+    event ReceiptAnchored(
+        bytes32 indexed runId,
+        address indexed publisher,
+        bytes32 yamlHash,
+        bytes32 datasetCidHash,
+        bytes32 checkpointCidHash,
+        bytes32 evalReportHash,
+        uint64 timestamp
+    );
+
+    error ReceiptAlreadyExists(bytes32 runId);
+
+    function anchor(
+        bytes32 runId,
+        bytes32 yamlHash,
+        bytes32 datasetCidHash,
+        bytes32 checkpointCidHash,
+        bytes32 evalReportHash
+    ) external {
+        if (_receipts[runId].timestamp != 0) revert ReceiptAlreadyExists(runId);
+        Receipt memory r = Receipt({
+            yamlHash: yamlHash,
+            datasetCidHash: datasetCidHash,
+            checkpointCidHash: checkpointCidHash,
+            evalReportHash: evalReportHash,
+            publisher: msg.sender,
+            timestamp: uint64(block.timestamp)
+        });
+        _receipts[runId] = r;
+        emit ReceiptAnchored(
+            runId,
+            msg.sender,
+            yamlHash,
+            datasetCidHash,
+            checkpointCidHash,
+            evalReportHash,
+            r.timestamp
+        );
+    }
+
+    function get(bytes32 runId) external view returns (Receipt memory) {
+        return _receipts[runId];
+    }
+
+    function exists(bytes32 runId) external view returns (bool) {
+        return _receipts[runId].timestamp != 0;
+    }
+}

contracts/src/x402_receiver.sol

@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: Apache-2.0
+pragma solidity ^0.8.26;
+
+/// @title X402Receiver
+/// @notice Receives x402 HTTP 402 payment proofs and validates Algorand settlement
+///         hashes against an off-chain facilitator (parsec-wallet or Coinbase).
+/// @dev Cypherpunk2048 standard: immutable, no proxy, no admin. The facilitator
+///      address is fixed at deploy time; rotating it requires deploying a new
+///      contract.
+contract X402Receiver {
+    address public immutable facilitator;
+    bytes32 public immutable assetIdHash; // hash of (chain, asset_id) tuple, e.g. ("algorand", 203977300)
+
+    mapping(bytes32 => bool) public seen;
+
+    event PaymentValidated(
+        bytes32 indexed invoiceId,
+        bytes32 indexed settlementProof,
+        address indexed payer,
+        uint256 amount
+    );
+
+    error AlreadySeen(bytes32 invoiceId);
+    error NotFacilitator(address sender);
+
+    constructor(address facilitator_, bytes32 assetIdHash_) {
+        facilitator = facilitator_;
+        assetIdHash = assetIdHash_;
+    }
+
+    /// @notice The facilitator submits a settlement proof; this contract records it.
+    /// @dev Off-chain x402 flow: caller pays the Algorand asset, parsec-wallet
+    ///      builds an EIP-712 attestation, the facilitator submits it here.
+    function recordSettlement(
+        bytes32 invoiceId,
+        bytes32 settlementProof,
+        address payer,
+        uint256 amount
+    ) external {
+        if (msg.sender != facilitator) revert NotFacilitator(msg.sender);
+        if (seen[invoiceId]) revert AlreadySeen(invoiceId);
+        seen[invoiceId] = true;
+        emit PaymentValidated(invoiceId, settlementProof, payer, amount);
+    }
+}

contracts/test/MindxtrainRegistry.t.sol

@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: Apache-2.0
+pragma solidity ^0.8.26;
+
+import {Test} from "forge-std/Test.sol";
+import {MindXTrainRegistry} from "../src/mindxtrain_registry.sol";
+
+contract MindXTrainRegistryTest is Test {
+    MindXTrainRegistry registry;
+
+    function setUp() public {
+        registry = new MindXTrainRegistry();
+    }
+
+    function test_anchor_emits_event() public {
+        bytes32 runId = keccak256("run-1");
+        bytes32 yamlHash = keccak256("yaml");
+        bytes32 datasetHash = keccak256("dataset");
+        bytes32 checkpointHash = keccak256("checkpoint");
+        bytes32 evalHash = keccak256("eval");
+
+        vm.expectEmit(true, true, false, true);
+        emit MindXTrainRegistry.ReceiptAnchored(
+            runId,
+            address(this),
+            yamlHash,
+            datasetHash,
+            checkpointHash,
+            evalHash,
+            uint64(block.timestamp)
+        );
+        registry.anchor(runId, yamlHash, datasetHash, checkpointHash, evalHash);
+    }
+
+    function test_anchor_persists_receipt() public {
+        bytes32 runId = keccak256("run-2");
+        registry.anchor(
+            runId, keccak256("y"), keccak256("d"), keccak256("c"), keccak256("e")
+        );
+        MindXTrainRegistry.Receipt memory r = registry.get(runId);
+        assertEq(r.yamlHash, keccak256("y"));
+        assertEq(r.publisher, address(this));
+        assertGt(r.timestamp, 0);
+        assertTrue(registry.exists(runId));
+    }
+
+    function test_anchor_rejects_overwrite() public {
+        bytes32 runId = keccak256("run-3");
+        registry.anchor(
+            runId, keccak256("y"), keccak256("d"), keccak256("c"), keccak256("e")
+        );
+        vm.expectRevert(
+            abi.encodeWithSelector(MindXTrainRegistry.ReceiptAlreadyExists.selector, runId)
+        );
+        registry.anchor(
+            runId, keccak256("y2"), keccak256("d2"), keccak256("c2"), keccak256("e2")
+        );
+    }
+
+    function test_exists_false_for_unknown() public view {
+        assertFalse(registry.exists(keccak256("never-anchored")));
+    }
+
+    function testFuzz_anchor_distinct_run_ids(bytes32 a, bytes32 b) public {
+        vm.assume(a != b);
+        registry.anchor(a, bytes32(0), bytes32(0), bytes32(0), bytes32(0));
+        registry.anchor(b, bytes32(0), bytes32(0), bytes32(0), bytes32(0));
+        assertTrue(registry.exists(a));
+        assertTrue(registry.exists(b));
+    }
+}

docs/CHANGELOG.md

@@ -0,0 +1,200 @@
+# Changelog
+
+All notable changes to **mindxtrain** are documented in this file. The format
+follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this
+project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Changed
+
+- **Removed hackathon framing from the public surfaces.** README, `pyproject.toml`
+  description, the package docstring, `CLAUDE.md`, and the active docs (autotune /
+  architecture / development / coach / yaml_schema / HANDOFF / NAV) no longer frame
+  mindXtrain as a hackathon submission; deleted `docs/HACKATHON.md` and
+  `docs/hackathon_submission.md`. The frozen `docs/blueprints/` and dated dev-blog
+  posts are preserved unchanged as the historical record. README now leads with the
+  dcoach proof loop.
+
+### Added
+
+- **dcoach page + prompt-tools + decentralized panel.** New `/coach/dcoach` workflow
+  page: one-click **Imprint & Prove** (persona/skills + advanced toggles) streams the
+  proof loop live (`POST /coach/api/dcoach/run`, SSE) and shows the classroom before/after
+  recall, boardroom verdict, and autotune-feedback next-params. A read-only
+  **decentralized-network panel** (`GET /coach/api/decentralized`) maps Prime Intellect /
+  Templar SN3 / Nous Psyche / Gensyn / Pluralis to how mindXtrain fits (AOT-only ⇒ Verde-
+  compatible; BLAKE3 receipt; x402; AgenticPlace). New `/coach/prompts` **prompt-tools**
+  page: craft a system prompt + few-shot demos → test against a base model (no training) →
+  evaluate (`POST /coach/api/eval/prompt`) → **make permanent** via Modelfile. Docs:
+  [dcoach.md](dcoach.md).
+- **dcoach proof loop + clean-room eval tools.** Prove a CPU-trained model recalls
+  its training: `governance.proof_loop.run_proof_loop` chains compose-script → imprint-
+  train → probe before/after → **classroom** (`governance.classroom.evaluate_classroom`)
+  → **boardroom** decides → **autotune feedback** (`autotune.feedback`, nudges the next
+  run's params). New clean-room LlamaIndex-style evaluators (`eval.llama_evals`:
+  Semantic-Similarity / Correctness / Pairwise / Guideline — reuse existing embeddings +
+  the governance chat-judge). Endpoints `POST /coach/api/classroom/evaluate` and
+  `POST /coach/api/autotune/feedback`.
+
+- **Streaming chat + ollama controls** (Coach "Try the model" card). Responses now
+  **stream token-by-token** (the AI-SDK text-stream pattern over SSE) via
+  `POST /coach/api/chat/stream`, with a **model picker** (local models first, no more
+  silently-broken `:cloud` default) and **start/stop/status** for the local ollama
+  server (`/coach/api/ollama/{status,start,stop}`). Fixes the chat that returned no
+  visible response. Reference: `docs/Vercel AI SDK 6_ … .md`.
+
+- **Ollama Modelfile builder** (`mindxtrain.deploy.modelfile`) — render a valid
+  `Modelfile` from a typed `ModelfileSpec` (every instruction: FROM/SYSTEM/TEMPLATE/
+  ADAPTER/LICENSE/MESSAGE/REQUIRES + the full PARAMETER catalogue). A standalone Coach
+  window at `/coach/modelfile` exposes a toggle + input for every instruction and
+  parameter; `POST /coach/api/modelfile/{build,create}` render it and run `ollama create`.
+- **Default personas + skills** (`mindxtrain.data.personas`) — built-in personas
+  (codephreak / assistant / mentor) and toggleable **skills** (software engineer, platform
+  architect, bash, solidity) that mix in-domain exchanges into a script. The Create-script
+  card gains a persona picker + skill toggles; `derive_training_params` auto-tunes CPU
+  imprint epochs/grad_accum from the dataset size.
+
+- **Governance layer** (`mindxtrain.governance`) — classroom / boardroom / dojo,
+  a clean-room reimplementation of openmindx/openmind's Boardroom-consensus +
+  Dojo-evaluation behaviour. The **classroom** graduates an actor on its imprint;
+  the **boardroom** (any N role-based members) convenes on the promotion motion →
+  approved / rejected / disputed; the **dojo** settles a dispute with a panel of a
+  **prime number** of judges (odd prime ≥ 3 → no tie, always resolves). See
+  [`docs/governance.md`](governance.md).
+- **Model-backed boardroom/dojo** (`governance.panel`) — members + judges
+  deliberate with real models over any OpenAI-compatible backend
+  (`deliberate`, `model_ballot`, `model_judge_ballot`). Coach **Boardroom** card +
+  `GET /coach/api/boardroom/presets`, `POST /coach/api/boardroom/convene`,
+  `POST /coach/api/dojo/settle` (model deliberation runs off the event loop).
+
+### Fixed
+
+- **Imprint recall now measured under the trained conditioning.** `probe_recall` accepts an
+  optional `system` prompt and the proof loop passes the persona's system prompt to both the
+  before and after probes — matching the system turn the script rows train under. Without it
+  the adapter was probed out of its learned distribution, understating (even inverting) the
+  imprint; the CPU proof loop now reports a strong positive Δ (e.g. 0.066→0.247) and an
+  APPROVED verdict.
+
+## [1.0.0] — 2026-06-11
+
+First production release. **CPU training is active end-to-end**; the GPU
+(MI300X / consumer) path is code-complete and validated on CPU dry-run + unit
+tests, pending real ROCm hardware to execute. Honest per-module status is in
+[`docs/actualization_status.md`](actualization_status.md).
+
+### Added (1.0.0)
+
+- **Device-aware local-GPU lane** (`trl_local`,
+  `mindxtrain.train.backend_trl_cpu.run_trl_local`). Auto-detects a consumer GPU
+  (CUDA or ROCm Radeon, bf16/fp16) and falls back to CPU; `trl_cpu` is the
+  force-CPU wrapper. Recipe `mindx_fallback_qwen3_1_5b_local`. Honest limit:
+  integrated Vega/`gfx90c` APUs are unsupported and fall back to CPU.
+- **Verifiable training receipt.** Every operator/CPU run emits `manifest.json`
+  binding the frozen `AutotunePlan` hash to the checkpoint + config hashes
+  (`provenance.manifest.emit_receipt_for_run`); re-verified via `mindxtrain
+  receipt`, the `GET /coach/api/receipt/{run_id}` endpoint, and a Coach
+  "Verifiable receipt" card. Optional x402 metering gate on `/v1/training/jobs`.
+- **Actor / persona / script + imprint.** Author a training *script* for an
+  *actor* (`mindxtrain.data.scripts`), ingest as `source: local`, and measure the
+  persona **imprint** by recall before/after training
+  (`mindxtrain.eval.imprint`, `mindxtrain imprint`,
+  `POST /coach/api/imprint/score`). Coach **Create script** card +
+  `POST/GET /coach/api/datasets`. Recipe `mindx_persona_imprint_local`.
+- **mindX dream-cycle trigger** (`deploy.api_client.trigger_dream_ingestion`) —
+  hands an imprinted actor to mindX's `machine.dream` 8hr cycle via HTTP or an
+  inbox drop (clean-room: a pointer, never mindX code). `mindxtrain imprint
+  --trigger-dream`.
+- **Coach live-training diagnostics** — accurate depiction with accordion
+  compression: rolling loss-chart window, per-step metrics + log accordions with
+  honest "showing last N" counts, system-metric sparklines, chat re-probe.
+- **Autotune on real hardware** — runtime GPU-count autodetect (`rccl_probe`),
+  GEMM microbenchmark with timings (`gemm_probe`), `serve --to sglang`.
+- **Clean-room policy** codified in `CLAUDE.md` + `AGENTS.md`: reimplement/adapt
+  locally, never copy mindX/external source bytes.
+
+### Added (pre-1.0 foundation)
+
+- **mindX self-training loop.** `mindx_dreams` dataset source adapter walks
+  `<mindx>/data/memory/ltm/*/*_training.jsonl`, deduplicates by content hash,
+  and yields OpenAI-chat rows. Pure stdlib; no GPU/heavy deps to load
+  (`mindxtrain.data.sources.mindx_dreams`).
+- **CPU training lane** (`mindxtrain.train.backend_trl_cpu.run_trl_cpu`).
+  Real TRL SFT/LoRA on CPU, produces a real HF-format checkpoint compatible
+  with `quantize`/`receipt`/`publish`. Wired via `TrainingBackend = "trl_cpu"`.
+- **Public training-jobs API** at `/v1/training/jobs` with bearer auth
+  (`MINDXTRAIN_API_KEY`). Versioned facade over the same `RunRegistry` Coach
+  uses, so mindX agents and external clients become peer dispatchers
+  (`mindxtrain.operator.training_api`).
+- **mindX fallback-swap caller** (`mindxtrain.deploy.api_client.swap_mindx_fallback_model`).
+  PATCHes mindX's `/v1/config/fallback-model` so a freshly published HF Hub
+  checkpoint becomes mindX's active default without a source edit. Pairs with
+  the mindX-side endpoint shipped in AgenticPlace/mindX@ad8193ea3.
+- Two new YAML recipes: `mindx_fallback_qwen3_1_5b_sft_lora` (Qwen3-1.5B
+  LoRA on MI300X, the production target) and `mindx_fallback_qwen3_1_5b_cpu_smoke`
+  (SmolLM2-135M CPU smoke).
+- `.env.example`: `MINDXTRAIN_API_KEY` and `MINDXTRAIN_MINDX_HOME`.
+
+### Changed
+
+- Schema: `DataSource` extends to include `"mindx_dreams"`; `DataCfg.hf_id`
+  is now defaultable with a `model_validator` requiring it only for
+  `source: "hf"`; `DataCfg.path` added (used by `local` + `mindx_dreams`).
+- `HardwareCfg.gpus: Literal[0, 1, 8]` — `0` = CPU lane.
+- Production URL flipped from `mindx.pythai.net/hackathon` to
+  `mindx.pythai.net/coach`. Hackathon-era references preserved in
+  `HACKATHON.md` and the build-in-public posts for the historical record.
+- CI: ruff scoped to `mindxtrain/ tests/`; mypy points at the canonical
+  `mindxtrain/config mindxtrain/provenance` (fixing stale paths). Container
+  build added on push-to-main; GHCR publish on tag.
+
+### Notes
+
+- Adapter smoke against the real corpus saw 1051 unique rows in
+  `/home/hacker/mindX/data/memory` (corpus snapshot 2026-05-14).
+
+## [0.1.0] — 2026-05-06
+
+Initial public release. Submitted to the AMD × lablab.ai Developer Hackathon
+(build window May 4–10 2026, on-site finale May 9–10 in San Francisco).
+
+### Added
+
+- Single-package canonical layout per `docs/blueprints/mindxtrain2.md` §Part 4:
+  `mindxtrain/{cli,config,data,models,train,eval,autotune,operator,storage,provenance,deploy,budget}`.
+- CLI with 8 verbs: `init`, `bench`, `train`, `eval`, `quantize`, `serve`,
+  `publish`, `receipt` (`mindxtrain.cli.main`).
+- 60-second AOT autotune probe for AMD MI300X — the hackathon differentiator.
+  CK vs Triton attention selection + hipBLASLt GEMM heuristic + RCCL config
+  (`mindxtrain.autotune.{benchmark,attention_probe,gemm_probe,rccl_probe,plan}`).
+- Pydantic v2 `XTrainConfig` with discriminated union over 9 training methods
+  (full, lora, qlora, dpo, orpo, grpo, gspo, kto, cpt) (`mindxtrain.config.schema`).
+- 12 YAML training recipes covering Qwen3.5/Qwen3.6/Instella across
+  SFT-LoRA, full-FSDP, DPO, ORPO, GRPO, CPT, and VL.
+- Axolotl YAML compiler; alt backends (Unsloth, torchtune, Primus) wired as
+  dispatch stubs (`mindxtrain.train`).
+- Provenance manifest with BLAKE3 content addressing, ROCm/git/gfx capture,
+  and on-chain pointers (ERC-7857 INFT, Algorand ASA, ERC-8004 attestation)
+  (`mindxtrain.provenance`).
+- Operator FastAPI app exposing `/v1/chat/completions`, `/v1/agentic`, and
+  the interactive `/coach/` UI (`mindxtrain.operator`).
+- Pluggable inference backends: vLLM, OpenAI-compatible
+  (`mindxtrain.operator.backends`).
+- Pluggable storage providers: local fs, HF Hub, Lighthouse, IPFS
+  (`mindxtrain.storage`).
+- Foundry contracts for ERC-8004 attestation registry (`contracts/`).
+- Containerfiles + compose + k8s manifests for MI300X (`ops/`).
+
+### Notes
+
+- This release replaces the earlier 3-package layout
+  (`mindXtrain/`, `automindXtrain/`, `custmodel/`) with a single ordered
+  package `mindxtrain/`. Old import paths (`xtrain.*`, `automindx.*`,
+  `custmodel.*`) are not preserved — adopters must rewrite to `mindxtrain.*`.
+- Most operator and trainer surfaces ship as honest minimal stubs that raise
+  `NotImplementedError` for paths not yet implemented in the hackathon scope.
+  The autotune probe, config schema, manifest, recipes, and Axolotl compiler
+  are the production-ready paths.
+
+[0.1.0]: https://example.invalid/mindxtrain/releases/tag/v0.1.0

docs/HANDOFF.md

@@ -0,0 +1,309 @@
+# HANDOFF — what you need to do next
+
+This is the ordered checklist for taking the mindxtrain repo from "code is
+done" to "demo is live." Each step is concrete; check it off when finished.
+
+The repo state at handoff:
+
+- Single canonical package at `mindxtrain/` (12 subpackages, ~100 modules).
+- All stub `NotImplementedError` paths replaced with real Python (lazy imports
+  for heavyweight deps).
+- 112/112 tests pass on a CPU-only laptop (`uv sync` + `uv run pytest -q`).
+- Optional dep groups in `pyproject.toml`: `ml`, `eval`, `data`, `serve`,
+  `chain`, `obs`. Install only what you need.
+- 12 YAML training recipes wired through the CLI.
+- Coach UI (`/coach/`) serves all 12 recipes without GPU.
+
+---
+
+## 1. Local setup (no GPU; 10 minutes)
+
+```bash
+cd /home/hacker/Desktop/mindXtrain
+cp .env.example .env       # then edit .env to fill in HF_TOKEN, etc.
+uv sync                    # base install
+uv run pytest -q           # → 112 passed
+uv run mindxtrain --help   # all 9 verbs listed
+```
+
+**What goes in `.env`** (rest of the file is sane defaults):
+
+| Var | Where to get it |
+|---|---|
+| `HF_TOKEN` | https://huggingface.co/settings/tokens (write scope) |
+| `HF_HUB_USERNAME` | your HF handle |
+| `LIGHTHOUSE_API_KEY` | https://files.lighthouse.storage/dashboard/apikey |
+| `MINDXTRAIN_OPENAI_API_KEY` | optional; only if you want to use openai_compat backend |
+
+> **Optional (on-chain anchors):** `MINDXTRAIN_REGISTRY_ADDR` (ERC-8004 contract),
+> `MINDXTRAIN_FACILITATOR_URL` (x402 facilitator). The publish path skips
+> these gracefully if unset.
+
+## 2. Provision the MI300X droplet (sign-up + 30 min)
+
+> **Fast path (Coach UI):** if you've populated `GITHUB_TOKEN`,
+> `AMD_DEV_CLOUD_TOKEN`, and `AMD_DEV_CLOUD_SSH_KEY_ID` in `.env`, you can skip
+> the manual SSH dance entirely:
+>
+> 1. `uv run uvicorn mindxtrain.operator.app:app --port 8080`
+> 2. Open <http://localhost:8080/coach/>, scroll to step 6 ("Deploy").
+> 3. Click ① **Push to GitHub** → ② **Provision MI300X droplet**. The droplet
+>    boots, cloud-init clones the repo from the SHA you just pushed, pulls the
+>    container, and runs `mindxtrain bench` automatically. All output streams
+>    live in the browser via SSE.
+>
+> Equivalent CLI: `mindxtrain github push && mindxtrain droplet provision`.
+>
+> The manual sequence below is preserved for scripted / CI use and as a
+> fallback when the Coach UI isn't available.
+
+```bash
+# Sign up at https://devcloud.amd.com — request a single MI300X.
+# Wait for the droplet (typically same-day).
+# SSH in:
+ssh ubuntu@<droplet-ip>
+
+# Install podman if missing:
+sudo apt-get update && sudo apt-get install -y podman podman-compose
+
+# Pull the canonical training container:
+podman pull docker.io/rocm/primus:v26.2
+
+# Snapshot the digest into the repo so others can reproduce:
+podman inspect --format '{{index .RepoDigests 0}}' rocm/primus:v26.2 \
+  | tee -a ops/containerfiles/digest.lock
+
+# Verify the GPU is visible:
+podman run --rm --device=/dev/kfd --device=/dev/dri rocm/primus:v26.2 \
+  rocminfo | head -50
+# → should show gfx942, 192 GB HBM3
+```
+
+> **Cost watch:** $1.99/hr × planned hours. Budget ~$30 for the full demo
+> pipeline (~15 GPU-hours). Leave the droplet **stopped** when not actively
+> training.
+
+## 3. Install heavyweight deps inside the container
+
+```bash
+# On the MI300X:
+git clone <your-repo-url> /workspace/mindxtrain
+cd /workspace/mindxtrain
+podman run -it --rm \
+  --device=/dev/kfd --device=/dev/dri \
+  -v /workspace/mindxtrain:/workspace/mindxtrain \
+  -w /workspace/mindxtrain \
+  rocm/primus:v26.2 bash
+
+# Inside the container:
+pip install -e ".[ml,eval,data,obs]"
+# (skip `serve` and `chain` until you need them — they pull large wheels)
+```
+
+## 4. Run the autotune probe (real, ~60 s)
+
+```bash
+mindxtrain bench --gpu 0 --out plan.json
+cat plan.json | jq '.attention_backend, .gemm_heuristic, .rccl_config'
+# → "ck", "hipblaslt_default", "1gpu_noop"
+```
+
+Snapshot `plan.json` into the repo so the run is reproducible:
+
+```bash
+cp plan.json ops/k8s/plan-mi300x.json
+git add ops/k8s/plan-mi300x.json
+git commit -m "snapshot autotune plan from mi300x"
+```
+
+## 5. Train + eval + quantize (~ 2 hours total for the demo recipe)
+
+```bash
+# Pick a recipe: instella_3b_lora is the AMD-on-AMD demo path (~30 min).
+# Or qwen3_8b_sft_lora for the Qwen side prize (~75 min).
+mindxtrain init --template instella_3b_lora --out run.yaml
+
+# Optional: edit run.yaml for your project name, dataset, output path.
+$EDITOR run.yaml
+
+# Dataset prep (pulls + dedupes + tokenizes + packs):
+mindxtrain dataset prep run.yaml --out ./out/dataset
+
+# Training:
+mindxtrain train run.yaml --plan plan.json
+# → ./out/runs/<run_name>/checkpoint/
+
+# Evaluation (MMLU subset):
+mindxtrain eval run.yaml
+# → ./out/runs/<run_name>/eval/lm_eval.json
+
+# Quantize to FP8:
+mindxtrain quantize run.yaml
+# → ./out/runs/<run_name>/quantized/
+```
+
+If `mindxtrain train` fails with `accelerate not found`: you forgot
+`pip install -e ".[ml]"` inside the container (step 3).
+
+## 6. Build the manifest + verify
+
+```bash
+# Generate the provenance manifest by hashing every artifact:
+uv run python -c "
+from pathlib import Path
+from mindxtrain.config.loader import load_config
+from mindxtrain.provenance.manifest import emit_receipt, ProvenanceHashes
+cfg = load_config('run.yaml')
+run = Path('./out/runs') / cfg.meta.run_name
+m = emit_receipt(
+    cfg,
+    cfg.meta.run_name,
+    config_yaml_path=Path('run.yaml'),
+    dataset_manifest_path=run / 'dataset_manifest.json',
+    checkpoint_dir=run / 'checkpoint',
+    eval_json_path=run / 'eval/lm_eval.json',
+)
+out = run / 'manifest.json'
+out.write_text(m.model_dump_json(indent=2))
+print(out)
+"
+
+# Verify it round-trips:
+mindxtrain receipt ./out/runs/<run_name>/manifest.json --config run.yaml
+# → all BLAKE3 fields = true (config, checkpoint, autotune_plan; dataset/eval if present)
+```
+
+> **Auto-emitted receipts (operator + CPU lane).** Runs launched through the
+> operator — Coach UI or `POST /v1/training/jobs` — now write `manifest.json`
+> automatically at completion via `provenance.manifest.emit_receipt_for_run`,
+> alongside `config.snapshot.yaml` and `autotune_plan.json` in the run dir. The
+> receipt **binds the frozen AutotunePlan hash to the checkpoint hash** — this is
+> the AOT artifact that makes a run bitwise-verifiable (cf. Verde/RepOps). The
+> Coach "Verifiable receipt" card re-checks it live; `mindxtrain receipt` does the
+> same from a shell. The manual `emit_receipt` above remains the full GPU path
+> (dataset + eval JSON included). On MI300X, also snapshot the AOTriton /
+> hipBLASLt tuning caches next to `autotune_plan.json` so the compiled artifact —
+> not just the plan — is reproducible across machines.
+
+## 7. Publish (HF Hub + Lighthouse + mindX register)
+
+```bash
+# Push to HF (uses HF_TOKEN; private=False for the demo):
+mindxtrain publish run.yaml --manifest ./out/runs/<run_name>/manifest.json
+# → updates manifest.json in-place with hf_repo_id + lighthouse_cid
+```
+
+If `LIGHTHOUSE_API_KEY` is unset, the pin step skips gracefully and the
+manifest gets a `cid://stub-…` placeholder.
+
+## 8. Deploy contracts (optional)
+
+The demo can ship without on-chain anchors. Do these once, when ready:
+
+```bash
+cd contracts
+forge install
+forge test                       # local Foundry tests pass
+forge script script/Deploy.s.sol \
+  --rpc-url $MINDXTRAIN_BASE_RPC_URL \
+  --private-key $DEPLOYER_KEY \
+  --broadcast
+# → records contract address; paste into .env as MINDXTRAIN_REGISTRY_ADDR
+```
+
+Once `MINDXTRAIN_REGISTRY_ADDR` is set, `mindxtrain.provenance.erc8004.broadcast_attestation`
+can anchor the manifest BLAKE3 on-chain.
+
+## 9. Serve the model + wire the production URL
+
+The production URL is `https://mindx.pythai.net` — the Coach UI is at `/coach/`
+and the public training-jobs API is at `/v1/training/jobs`.
+
+```bash
+# Inside the rocm/vllm-dev container:
+podman-compose -f ops/compose/compose_dev.yaml up -d
+# → vLLM-ROCm at :8000, mindxtrain operator FastAPI at :8080
+
+# Verify locally:
+curl http://localhost:8080/coach/api/health
+# → {"coach_version":"0.1.0", "recipes_available":>=14, ...}
+
+# Public training-jobs API smoke (bearer auth via MINDXTRAIN_API_KEY):
+curl -X POST http://localhost:8080/v1/training/jobs \
+  -H "Authorization: Bearer $MINDXTRAIN_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"recipe":"mindx_fallback_qwen3_1_5b_cpu_smoke"}'
+# → {"job_id":"...", "status":"running", "backend":"trl_cpu", ...}
+
+# Reverse-proxy mindx.pythai.net → MI300X:8080 (Caddy/Cloudflare).
+```
+
+Once the proxy is live, `curl https://mindx.pythai.net/coach/api/health`
+returns 200 from the public internet.
+
+## 10. Publish & demo
+
+```bash
+# Push code:
+git push origin main
+
+# End-to-end demo walk-through:
+#   1. mindxtrain init  → show CLI verbs
+#   2. mindxtrain bench → 60-second autotune (the differentiator)
+#   3. mindxtrain train → timelapse of training
+#   4. mindxtrain quantize → FP8 weights
+#   5. curl /v1/chat/completions → live inference
+#   6. mindxtrain receipt → BLAKE3 reverify
+#   7. Open /coach/ → click through the UI
+#   8. Open /coach/dcoach → Imprint & Prove (CPU recall proof)
+```
+
+## 11. Quality gates (run before every push)
+
+```bash
+uv run ruff check .
+uv run mypy mindxtrain/config mindxtrain/provenance
+uv run pytest -q       # → 112 passed
+```
+
+All three must pass before pushing to `main`. CI runs the same gates on the
+`main` branch.
+
+---
+
+## What's still TODO
+
+These paths are wired but require runtime/contracts/services to actually
+flow end-to-end:
+
+- **x402 metering** (`mindxtrain.provenance.x402`) — wired to httpx, needs
+  a deployed facilitator URL.
+- **ERC-8004 broadcast** (`mindxtrain.provenance.erc8004.broadcast_attestation`)
+  — needs deployed attestation registry + signer key.
+- **BANKON ENS** allocation (`mindxtrain.provenance.algorand.allocate_ens_subname`)
+  — needs the BANKON allocation service deployed.
+- **AgenticPlace listing** (`mindxtrain.deploy.api_client.list_on_agenticplace`)
+  — needs `agenticplace.pythai.net` live.
+- **mindX agent register** (`mindxtrain.deploy.api_client.register_with_mindx`)
+  — needs `mindx.pythai.net/v1/agents` live.
+
+The framework itself ships as production-ready Apache-2.0; the integrations
+above are paid/external services you stand up at your own pace.
+
+---
+
+## Quick reference
+
+| What | Where |
+|---|---|
+| All CLI verbs | `mindxtrain --help` |
+| All recipes | `mindxtrain init --list` |
+| Coach UI | http://localhost:8080/coach/ |
+| Per-module status | `docs/actualization_status.md` |
+| Architecture | `docs/architecture.md` |
+| Autotune detail | `docs/autotune.md` |
+| Coach detail | `docs/coach.md` |
+| CLI reference | `docs/cli.md` |
+| YAML schema | `docs/yaml_schema.md` |
+| dcoach proof loop | `docs/dcoach.md` |
+| Frozen blueprints | `docs/blueprints/{mindXtrain,mindXtrain2}.md` |

docs/LICENSE-NOTICE.md

@@ -0,0 +1,7 @@
+# Licensing notice for the lablab.ai / AMD Developer Hackathon
+
+This project is licensed under the Apache License 2.0 (see [LICENSE](../LICENSE)).
+
+The Apache 2.0 License is fully MIT-compatible: any code in this repository may be relicensed under MIT terms by a downstream consumer who keeps the original Apache 2.0 NOTICE and copyright attribution intact, in accordance with §4(d) of the Apache License.
+
+This statement satisfies the lablab.ai hackathon submission requirement that entries be released under an open-source license that is MIT-compatible.

docs/NAV.md

@@ -0,0 +1,50 @@
+# mindXtrain Documentation Index
+
+Every doc lives in `docs/`. The only Markdown at the repo root is `README.md` (entry
+point) plus `CLAUDE.md` / `AGENTS.md` (agent-tooling entrypoints, required at root).
+Start at [Quickstart](quickstart.md); operators running the demo read [HANDOFF.md](HANDOFF.md).
+
+## Getting started
+
+- [Quickstart](quickstart.md) — install (with optional-dep groups), init, bench, train.
+- [HANDOFF.md](HANDOFF.md) — **operator checklist**: local setup → MI300X provision → train/eval/quantize → publish → contracts → deploy.
+
+## Architecture & invariants
+
+- [Architecture](architecture.md) — the 5-layer single-package layout + MI300X invariants + data flow.
+- [Development workflow](development.md) — toolchain, optional-deps, lazy-import pattern, invariants, **training lanes** (CPU / local-GPU / MI300X), how to add recipes/backends/methods.
+- [Actualization status](actualization_status.md) — per-module map of what's real vs. needs `--extra` vs. v1.0.0 CPU-active / GPU-pending / stub.
+- [Autotune deep-dive](autotune.md) — the 60-second AOT probe (the differentiator).
+
+## Coach UI & training workflow
+
+- [Coach UI](coach.md) — the interactive `/coach/` operator UI: create-script (personas + skills), live-training diagnostics, verifiable receipt, streaming chat + ollama controls, Modelfile builder.
+- [dcoach](dcoach.md) — `/coach/dcoach`: prove a CPU-trained model recalls its training (imprint → classroom → boardroom → autotune feedback), clean-room llama-style eval tools, prompt-tools page, and how mindXtrain fits decentralized training.
+- [Governance](governance.md) — classroom (graduation) / boardroom (any-N consensus) / dojo (prime-N dispute settlement), model-backed deliberation.
+
+## Decentralized training landscape (2026)
+
+- [Decentralized training deep-dive](decentralized-training-deep-dive-2026.md) — Prime Intellect / Nous Psyche / Gensyn / Templar / Pluralis, the DiLoCo/SparseLoCo algorithms, verification (TOPLOC / Verde / Gauntlet), and where mindXtrain fits (AOT-only = verifiable).
+- [LLM training-stack landscape](mindxtrain-llm-training-landscape-2026.md) — the open-source training/eval/quantize stack survey anchored on mindXtrain.
+- [Vercel AI SDK 6 deep-dive](<Vercel AI SDK 6_ A Framework-Agnostic Deep Dive (June 2026).md>) — the streaming/agent toolkit the Coach chat patterns after (clean-room, vanilla JS).
+
+## Reference
+
+- [CLI reference](cli.md) — every `mindxtrain` verb with synopsis, options, exit codes.
+- [YAML schema](yaml_schema.md) — every field of the 10-section `XTrainConfig`.
+- [Benchmarks](benchmarks.md) — target metrics + framework comparison.
+- [CHANGELOG](CHANGELOG.md) — version history (current: v1.0.0).
+- [LICENSE-NOTICE](LICENSE-NOTICE.md) — Apache-2.0 + MIT-compatibility statement.
+
+## Source briefs (`blueprints/`)
+
+The frozen design briefs the project was built against — historical specification; for
+current state read the docs above. Do not edit.
+
+- [`blueprints/mindXtrain.md`](blueprints/mindXtrain.md) — operating brief; three-track pitch, day-by-day execution.
+- [`blueprints/mindXtrain2.md`](blueprints/mindXtrain2.md) — technical reference; canonical Part 4 layout.
+- [`blueprints/mindXtrain_ Production Blueprint for the AMD and lablab.ai Hackathon.md`](<blueprints/mindXtrain_ Production Blueprint for the AMD and lablab.ai Hackathon.md>) — repo skeleton, hero recipe, immutable registry stub.
+
+## On-chain
+
+- [`contracts/README.md`](../contracts/README.md) — Foundry workspace for the immutable run-receipt registry + x402 receiver.

docs/Vercel AI SDK 6_ A Framework-Agnostic Deep Dive (June 2026).md

@@ -0,0 +1,340 @@
+# The Vercel AI SDK: A Framework-Agnostic Deep Dive (AI SDK 6, June 2026)
+
+## TL;DR
+- The Vercel AI SDK is an Apache-2.0-licensed TypeScript toolkit (npm package `ai`); the current major is **AI SDK 6** (`ai@6.0.x` — 6.0.199 per the vercel/ai GitHub releases page dated 9 Jun, 6.0.202 listed on npm). Its core layer (AI SDK Core) is fully framework-agnostic and runs in any JS runtime — Node.js 18+, Deno, Bun, edge — with zero Vercel hosting lock-in.
+- It gives you one unified API (`generateText`, `streamText`, `generateObject`/`streamObject`, `embed`/`embedMany`, tools, the `ToolLoopAgent` agent loop, MCP client, middleware, image/speech/transcription) across providers; you can point it at sovereign self-hosted inference (Ollama, vLLM, llama.cpp, LM Studio) via `@ai-sdk/openai-compatible` using direct provider keys, no gateway required.
+- For a clean-room backend: `npm i ai @ai-sdk/anthropic @ai-sdk/openai @ai-sdk/openai-compatible zod`, set `ANTHROPIC_API_KEY`/`OPENAI_API_KEY`, and you have working `generateText`/streaming in a plain Node script or any Hono/Express/Fastify server in minutes.
+
+## Key Findings
+- **Version reality check:** Despite stale knowledge suggesting v4/v5, Vercel's official blog announced **AI SDK 5 on July 31 2025** ("v5 is a stable, production release"), and **v6 followed in late 2025** with further additions; as of June 2026 both `ai@6.0.x` and a maintained `ai@5.0.x` line are published in parallel (GitHub releases shows `ai@6.0.199` and `ai@5.0.197` published the same time). Use v6 for new projects.
+- **The Agent class was renamed.** v5's `Experimental_Agent` is now the stable **`ToolLoopAgent`** in v6, and `system` → `instructions`. `Agent` is now an *interface* you can implement (e.g. Workflow DevKit's `DurableAgent`).
+- **Tool API changed across v5/v6:** use `inputSchema` (not `parameters`), and control multi-step loops with `stopWhen: stepCountIs(n)` (not `maxSteps`).
+- **Sovereignty is well-supported:** direct provider packages auto-read env keys, the AI Gateway is entirely optional, and `@ai-sdk/openai-compatible` cleanly targets local inference servers. License is Apache-2.0.
+- **v6 deprecations to note:** `generateObject`/`streamObject` are now *deprecated* in favor of `generateText`/`streamText` with an `output` setting (though still fully functional); `CoreMessage` removed in favor of `ModelMessage`; `convertToCoreMessages` → `convertToModelMessages` (now async).
+
+## Details
+
+### 1. Architecture & Philosophy
+
+The AI SDK is "the AI Toolkit for TypeScript… a free open-source library for building AI-powered applications and agents" (vercel/ai). It is organized in layers:
+
+- **AI SDK Core** (`ai` package): the unified, framework-agnostic API for text/object generation, embeddings, tools, agents, image/speech. This is what a backend/agent engineer uses. It runs in *any* JavaScript runtime — Node.js, Deno, Bun, edge runtimes, plain backend services — because it depends only on standard web primitives (`fetch`, `ReadableStream`, SSE).
+- **AI SDK UI** (`@ai-sdk/react`, `@ai-sdk/vue`, `@ai-sdk/svelte`, `@ai-sdk/angular`): framework hooks (`useChat`, `useCompletion`, `useObject`). *Optional* — irrelevant for headless backends.
+- **Provider packages** (`@ai-sdk/*`): each adapts a vendor API to the SDK's Language Model Specification (currently the v3 spec in SDK 6; the underlying model interface evolved to LanguageModelV2 in v5).
+
+**The unified provider abstraction model.** Every provider package exposes a factory (`openai('gpt-5.4')`, `anthropic('claude-opus-4-6')`) returning a `LanguageModel` that conforms to a single spec. Switching providers is a one-line change. The v5 LanguageModelV2 redesign made all model outputs "content parts" (text, reasoning, tool calls, sources, files) in one ordered array, which is why reasoning models, multimodal, and computer-use agents all work through one interface.
+
+**Package structure / first-party providers** (all under `@ai-sdk/`): `openai`, `anthropic`, `google` (Generative AI), `google-vertex`, `mistral`, `groq`, `amazon-bedrock`, `azure`, `xai` (Grok), `deepseek`, `togetherai`, `cohere`, `fireworks`, `cerebras`, `deepinfra`, `perplexity`, `replicate`, `fal`, `luma`, `elevenlabs`, `assemblyai`, `deepgram`, `gladia`, `lmnt`, `hume`, `revai`, `baseten`, `huggingface`, `vercel` (v0), plus the crucial **`@ai-sdk/openai-compatible`** generic adapter and **`@ai-sdk/gateway`**.
+
+**Community / self-hostable providers** (implement the Language Model Specification): **Ollama** (`ollama-ai-provider-v2` and `ai-sdk-ollama` — the latter is a v6 provider built on the official `ollama` package, with `ai-sdk-ollama@^2` for v5), **llama.cpp**, **LM Studio** and **NVIDIA NIM** (documented under the openai-compatible umbrella), Cloudflare Workers AI, OpenRouter, Portkey, FriendliAI, LangDB, Browser AI (WebLLM/Transformers.js for in-browser models), and many more. For the sovereignty-minded: *any* server implementing the OpenAI API spec works through `@ai-sdk/openai-compatible` with no dedicated package.
+
+**v4 → v5 → v6 breaking changes (the ones that matter for backend code):**
+- **v4 → v5** (July 31 2025, major architectural overhaul): `UIMessage` and `ModelMessage` became separate types (conversion is now explicit via `convertToModelMessages`); streaming switched from a custom protocol to native **Server-Sent Events**; tools use `inputSchema`/`outputSchema` instead of `parameters`/`result`; `maxTokens` → `maxOutputTokens`; multi-step uses `stopWhen` (the `maxSteps` parameter was removed from `useChat`); `.reasoning` → `.reasoningText`; `mimeType` → `mediaType`; new `Experimental_Agent` class; speech/transcription added; Zod 4 supported. Codemods: `npx @ai-sdk/codemod@latest migrate`.
+- **v5 → v6** (late 2025): `Experimental_Agent` → **`ToolLoopAgent`** (and `system` → `instructions`); **`generateObject`/`streamObject` deprecated** in favor of `generateText`/`streamText` + `output`; **`CoreMessage` removed** (use `ModelMessage`); `convertToCoreMessages` → `convertToModelMessages` (now **async**); embedding methods `textEmbeddingModel`/`textEmbedding` → `embeddingModel`/`embedding`; `strictJsonSchema` on by default; `structuredOutputs` provider option removed (use `strictJsonSchema`); Azure provider defaults to the Responses API; tool UI helper renames (`isToolUIPart` → `isStaticToolUIPart`, etc.); a deprecation-warning logger (disable with `AI_SDK_LOG_WARNINGS=false`). Vercel describes v6 migration as "intentionally simple" with codemods (`npx @ai-sdk/codemod upgrade`).
+
+**Node requirement:** AI SDK 6 requires **Node.js 18+**. The official Getting Started: Node.js docs state "Node.js 18+ and pnpm installed on your local development machine," the npm README states "You will need Node.js 18+," and the repo `package.json` engines field is `"node": "^18.0.0 || ^20.0.0 || ^22.0.0"` — so Node 18/20/22 are the supported majors with 18 as the floor.
+
+### 2. Detailed Capability List
+
+**Text generation & streaming.** `generateText({ model, prompt | messages, system, tools, stopWhen, ... })` returns `{ text, reasoning, reasoningText, steps, toolCalls, toolResults, usage, finishReason, ... }`. `streamText(...)` returns a result exposing `textStream` (async iterable of text chunks), `fullStream` (typed parts: text-delta, reasoning, tool-input-start/delta, tool-call, tool-result, source, finish), plus `onChunk`, `onFinish`, `onError`, `onStepFinish`, and experimental lifecycle callbacks (`experimental_onStart`, `experimental_onStepStart`, `experimental_onToolCallStart`/`Finish`). Streaming uses backpressure — you must consume the stream for it to finish. Response helpers: `toUIMessageStreamResponse()`, `pipeUIMessageStreamToResponse(res)`, `toTextStreamResponse()`, `pipeTextStreamToResponse(res)`.
+
+**Structured output.** Two routes: (a) the still-functional but v6-deprecated `generateObject`/`streamObject({ model, schema, output })` with `output: 'object' | 'array' | 'enum' | 'no-schema'`; (b) the v6-preferred `generateText`/`streamText` with the `output` setting and the `Output` helper: `Output.object({ schema })`, `Output.array({ element })`, `Output.choice({ options })` (enum/classification), `Output.json()` (unstructured). Schemas may be Zod, Valibot, or JSON Schema (`jsonSchema()`). `streamObject`/array exposes `partialObjectStream` and `elementStream` (each element validated as it completes). Crucial caveat: structured output **counts as a step**, so when combining with tools, increase `stopWhen` accordingly.
+
+**Tool calling / function calling.** Define tools with the `tool()` helper (needed for TypeScript to infer `execute` arg types from `inputSchema`):
+```ts
+weather: tool({
+  description: 'Get the weather in a location',
+  inputSchema: z.object({ location: z.string() }),
+  execute: async ({ location }, { abortSignal }) => ({ location, temperature: 72 }),
+})
+```
+`execute` is optional (omit to forward calls to a client/queue). v6 adds: `needsApproval` (human-in-the-loop, boolean or function of input), per-tool `strict` mode, `toModelOutput` for flexible tool outputs, input examples, and `dynamicTool()` for runtime-defined tools. Multi-step loops: `stopWhen` accepts `stepCountIs(n)` (default `stepCountIs(20)`), `hasToolCall(name)`, `isLoopFinished()` (no limit), custom `StopCondition` functions, or an array (stops on any). `toolChoice: 'auto' | 'required' | 'none' | { type:'tool', toolName }`. Provider-executed tools exist (web search, code execution, memory, computer use). The abort signal is forwarded into `execute`.
+
+**Agents.** `ToolLoopAgent` (v6) encapsulates model + instructions + tools + loop control into a reusable object usable across chat UIs, background jobs, API endpoints, and CLI daemons:
+```ts
+const agent = new ToolLoopAgent({ model, instructions, tools, stopWhen: stepCountIs(10), prepareStep });
+const result = await agent.generate({ prompt });       // GenerateTextResult
+const stream = agent.stream({ prompt });                // StreamTextResult
+```
+Loop control: `stopWhen` (when to stop) and **`prepareStep`** (called before each step; can change model, tools, toolChoice, messages — used for context compression, model-switching by complexity, dynamic tool gating). v6 also adds `callOptionsSchema` + `prepareCall` for type-safe per-call options (e.g. inject RAG context once per call, select model by tier). Subagents are just a `ToolLoopAgent` invoked inside another agent's tool `execute`. For full control, hand-roll the loop with `generateText` + your own while-loop.
+
+**Embeddings & RAG.** `embed({ model, value })` → `{ embedding, usage }`; `embedMany({ model, values, maxParallelCalls })` → `{ embeddings, usage }` (auto-chunks large batches). `cosineSimilarity(a, b)` for ranking. v6 also adds a `rerank()` function. The canonical mini-RAG pattern: chunk → `embedMany` → store `{embedding, value}` → at query time `embed` the query, sort chunks by `cosineSimilarity`, inject top-k into the prompt. Production guides use pgvector or Upstash Vector as the store.
+
+**Image generation.** `generateImage({ model, prompt, size })` → `{ images }` (experimental, also exported as `experimental_generateImage`). v6 adds image editing/inpainting (`prompt: { text, images, mask }`) via the openai-compatible provider's `/images/edits`.
+
+**Speech & transcription (experimental).** `generateSpeech({ model: openai.speech('tts-1'), text, voice })` → `{ audio }`; `transcribe({ model: openai.transcription('whisper-1'), audio })` → `{ text, segments, language, durationInSeconds }`. Imported as `experimental_generateSpeech`/`experimental_transcribe`. Providers include OpenAI, ElevenLabs, Deepgram, AssemblyAI, Gladia, LMNT, Hume, Rev.ai. `audio` accepts Uint8Array/ArrayBuffer/Buffer/base64/URL.
+
+**Multimodal inputs.** Messages support `ImagePart`, `FilePart` (PDFs, files) alongside `TextPart`; images/files accept `string | Uint8Array | Buffer | ArrayBuffer | URL` with a `mediaType`.
+
+**Reasoning models.** Configure via `providerOptions`. For Anthropic extended thinking: `providerOptions: { anthropic: { thinking: { type: 'enabled', budgetTokens: 12000 } } }` (an `effort: 'low'|'medium'|'high'` option also exists; both can be combined). Access via destructured `reasoning`/`reasoningText`, or in streaming via `fullStream` parts. For OpenAI o-series/GPT-5: `providerOptions: { openai: { reasoningEffort: 'low', reasoningSummary: 'auto' } }`, with reasoning token counts at `providerMetadata.openai.reasoningTokens`. For models that wrap reasoning in `<think>` tags (DeepSeek R1, Magistral), use `extractReasoningMiddleware({ tagName: 'think' })`:
+```ts
+const model = wrapLanguageModel({ model: yourModel, middleware: extractReasoningMiddleware({ tagName: 'think' }) });
+const { text, reasoningText } = await generateText({ model, prompt: 'What is 15 * 24?' });
+```
+
+**Middleware.** `wrapLanguageModel({ model, middleware })` returns an enhanced model. A middleware (type `LanguageModelV3Middleware` in v6) implements any of `transformParams`, `wrapGenerate`, `wrapStream` — model-agnostic logging, caching, guardrails, RAG injection, rate-limiting. Multiple middlewares compose in order (applied innermost-last). Built-ins: `extractReasoningMiddleware`, `simulateStreamingMiddleware`, `defaultSettingsMiddleware`, `addToolInputExamplesMiddleware`, `extractJsonMiddleware`. Community: `@ai-sdk-tool/parser` (`hermesToolMiddleware`, `gemmaToolMiddleware`) adds tool-calling to local models lacking native function calling — directly relevant to self-hosted deployments.
+
+**Provider registry & custom providers.** `createProviderRegistry({ anthropic, openai, … })` lets you reference models by `providerId:modelId` string at runtime (custom separator supported) — useful for runtime model selection, A/B testing, and fallback routing. `customProvider({ languageModels, fallbackProvider })` creates aliases/preconfigured settings and can restrict the model set.
+
+**Telemetry.** OpenTelemetry-based via `experimental_telemetry: { isEnabled: true, functionId, recordInputs, recordOutputs }` on any generate/stream call. Emits standard `gen_ai.*` and AI-SDK-specific `ai.*` spans (model calls, `ai.toolCall`, etc.). Works with any OTel backend; for non-Next.js (Express/Fastify/Hono/plain Node) initialize the OTel Node SDK directly (e.g. `@opentelemetry/sdk-node` + an OTLP exporter, or `@pydantic/logfire-node`, or `@langfuse/otel`'s `LangfuseSpanProcessor`).
+
+**Error handling, retries, abort, timeouts.** `maxRetries` on all functions; `abortSignal` accepted by generate/stream/embed/transcribe and forwarded into tools; `streamText` puts errors into the stream (use `onError`) rather than throwing, to avoid crashing servers; typed errors (`AI_NoSpeechGeneratedError`, `MCPClientError`, etc.).
+
+**Streaming protocols & frameless consumption.** Native SSE. The **UI Message Stream** protocol (set header `x-vercel-ai-ui-message-stream: v1` for custom backends) carries typed parts; the **text stream** is plain text. To consume *without any frontend framework*: iterate `result.textStream`/`result.fullStream` in a CLI/daemon; or serve over HTTP with `pipeUIMessageStreamToResponse(res)` (Node `http`), `result.toUIMessageStreamResponse()`/`toTextStreamResponse()` (Hono/edge/Web `Response`), or Hono's `stream`/`streamSSE` helpers. `createUIMessageStream({ execute })` + `writer.write`/`writer.merge` lets you emit custom data parts. `readUIMessageStream` converts a chunk stream to an async-iterable of `UIMessage`s on the client side.
+
+**Prompt management.** `system` prompt, `prompt` (string), or `messages` array of `ModelMessage` (`SystemModelMessage`/`UserModelMessage`/`AssistantModelMessage`/`ToolModelMessage`, each with typed content parts). `UIMessage` (client-facing, has a `parts` array) is distinct from `ModelMessage` (sent to the LLM); convert with the async `convertToModelMessages(uiMessages)`.
+
+**Caching / rate limiting.** Implemented as middleware (cache by hashed params, short-circuit identical calls) or via provider-native prompt caching (Anthropic, Bedrock). The cookbook ships local-caching and dynamic-prompt-caching middleware examples.
+
+### 3. Clean-Room Setup (framework-agnostic)
+
+```bash
+mkdir my-agent && cd my-agent
+npm init -y
+npm pkg set type=module
+npm i ai @ai-sdk/anthropic @ai-sdk/openai @ai-sdk/openai-compatible zod
+npm i -D typescript tsx @types/node
+npx tsc --init
+```
+`tsconfig.json`: use ESM-friendly settings — `"module": "ES2022"`, `"moduleResolution": "Bundler"` (or `"NodeNext"`), `"target": "ES2022"`, `"strict": true`. Node 18+ (20 or 22 recommended). Set env vars — provider packages auto-detect them: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, etc. (Gateway string-model usage instead reads `AI_GATEWAY_API_KEY`.)
+
+Run scripts with `npx tsx index.ts`. **Deno**: `deno run --allow-net --allow-env npm:tsx index.ts` or import via `npm:ai`. **Bun**: `bun add ai @ai-sdk/anthropic zod` then `bun index.ts`.
+
+### 4. Code Examples (current v6 API)
+
+**Hello world — generateText (plain Node script):**
+```ts
+import { generateText } from 'ai';
+import { anthropic } from '@ai-sdk/anthropic';
+
+const { text } = await generateText({
+  model: anthropic('claude-opus-4-6'),
+  prompt: 'Explain quantum entanglement in two sentences.',
+});
+console.log(text);
+```
+
+**streamText — consume in a CLI/daemon:**
+```ts
+import { streamText } from 'ai';
+import { openai } from '@ai-sdk/openai';
+
+const result = streamText({
+  model: openai('gpt-5.4'),
+  prompt: 'Write a haiku about distributed systems.',
+});
+for await (const chunk of result.textStream) process.stdout.write(chunk);
+console.log('\nusage:', await result.usage);
+```
+
+**Structured output (v6-preferred output setting):**
+```ts
+import { generateText, Output } from 'ai';
+import { anthropic } from '@ai-sdk/anthropic';
+import { z } from 'zod';
+
+const { output } = await generateText({
+  model: anthropic('claude-sonnet-4-5'),
+  output: Output.object({
+    schema: z.object({
+      title: z.string(),
+      severity: z.enum(['low', 'medium', 'high']),
+      tags: z.array(z.string()),
+    }),
+  }),
+  prompt: 'Classify this incident: database connection pool exhausted under load.',
+});
+console.log(output);
+```
+
+**Tool calling + multi-step loop:**
+```ts
+import { generateText, tool, stepCountIs } from 'ai';
+import { anthropic } from '@ai-sdk/anthropic';
+import { z } from 'zod';
+
+const { text, steps } = await generateText({
+  model: anthropic('claude-sonnet-4-5'),
+  stopWhen: stepCountIs(5),
+  tools: {
+    getBalance: tool({
+      description: 'Get the on-chain token balance for an address',
+      inputSchema: z.object({ address: z.string(), token: z.string() }),
+      execute: async ({ address, token }) => ({ address, token, balance: '1234.56' }),
+    }),
+  },
+  prompt: 'What is the USDC balance of 0xabc...?',
+});
+console.log(text, steps.length);
+```
+
+**ToolLoopAgent (reusable agent):**
+```ts
+import { ToolLoopAgent, stepCountIs } from 'ai';
+import { anthropic } from '@ai-sdk/anthropic';
+
+export const agent = new ToolLoopAgent({
+  model: anthropic('claude-sonnet-4-6'),
+  instructions: 'You are an autonomous backend agent. Use tools, then answer.',
+  tools: { /* ...tools... */ },
+  stopWhen: stepCountIs(20),
+  prepareStep: ({ stepNumber }) => (stepNumber === 0 ? { toolChoice: 'required' } : {}),
+});
+
+const result = await agent.generate({ prompt: 'Analyze the dataset and summarize.' });
+console.log(result.text);
+```
+
+**Embeddings + cosine similarity mini-RAG:**
+```ts
+import { embed, embedMany, cosineSimilarity, generateText } from 'ai';
+import { openai } from '@ai-sdk/openai';
+
+const chunks = essay.split('.').map(s => s.trim()).filter(Boolean);
+const { embeddings } = await embedMany({
+  model: openai.embeddingModel('text-embedding-3-small'),
+  values: chunks,
+});
+const db = embeddings.map((embedding, i) => ({ embedding, value: chunks[i] }));
+
+const { embedding: q } = await embed({
+  model: openai.embeddingModel('text-embedding-3-small'),
+  value: 'What did the author say about sovereignty?',
+});
+const top = db
+  .map(d => ({ ...d, score: cosineSimilarity(q, d.embedding) }))
+  .sort((a, b) => b.score - a.score)
+  .slice(0, 3)
+  .map(d => d.value)
+  .join('\n');
+
+const { text } = await generateText({
+  model: openai('gpt-5.4'),
+  system: `Answer using only this context:\n${top}`,
+  prompt: 'What did the author say about sovereignty?',
+});
+console.log(text);
+```
+
+**Self-hosted inference via openai-compatible (Ollama / vLLM / llama.cpp):**
+```ts
+import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
+import { generateText } from 'ai';
+
+const local = createOpenAICompatible({
+  name: 'local',
+  baseURL: 'http://localhost:11434/v1', // Ollama; vLLM/llama.cpp server: their /v1 URL
+  apiKey: 'ollama',                      // placeholder; many local servers ignore it
+});
+
+const { text } = await generateText({
+  model: local('qwen2.5-coder:7b'),
+  prompt: 'Refactor this function for readability.',
+});
+console.log(text);
+```
+(Note the `/v1` suffix is required — local servers expose the OpenAI-compatible API there, not at their native `/api` path. The dedicated `ai-sdk-ollama` provider is an alternative that adds tool-call reliability and JSON repair on top of the official `ollama` client.)
+
+**Provider registry with Anthropic + a local endpoint:**
+```ts
+import { createProviderRegistry, generateText } from 'ai';
+import { anthropic } from '@ai-sdk/anthropic';
+import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
+
+const registry = createProviderRegistry({
+  anthropic,
+  local: createOpenAICompatible({ name: 'local', baseURL: 'http://localhost:11434/v1', apiKey: 'ollama' }),
+});
+
+const { text } = await generateText({
+  model: registry.languageModel('local:llama3.3'),   // or 'anthropic:claude-sonnet-4-6'
+  prompt: 'Summarize the latest block.',
+});
+```
+
+**Middleware (reasoning extraction for a local DeepSeek-R1):**
+```ts
+import { wrapLanguageModel, extractReasoningMiddleware, generateText } from 'ai';
+import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
+
+const local = createOpenAICompatible({ name: 'local', baseURL: 'http://localhost:8000/v1', apiKey: 'x' });
+const model = wrapLanguageModel({
+  model: local('deepseek-r1'),
+  middleware: extractReasoningMiddleware({ tagName: 'think' }),
+});
+const { text, reasoningText } = await generateText({ model, prompt: 'What is 15 * 24?' });
+console.log({ reasoningText, text });
+```
+
+**HTTP endpoint — Hono (Web Response):**
+```ts
+import { serve } from '@hono/node-server';
+import { streamText } from 'ai';
+import { Hono } from 'hono';
+
+const app = new Hono();
+app.post('/', async c => {
+  const result = streamText({ model: 'openai/gpt-4o', prompt: 'Invent a holiday.' });
+  return result.toUIMessageStreamResponse();
+});
+serve({ fetch: app.fetch, port: 8080 });
+```
+
+**HTTP endpoint — plain Node `http` (no framework):**
+```ts
+import { streamText } from 'ai';
+import { createServer } from 'http';
+
+createServer(async (req, res) => {
+  const result = streamText({ model: 'openai/gpt-4o', prompt: 'Invent a holiday.' });
+  result.pipeUIMessageStreamToResponse(res);
+}).listen(8080);
+```
+(Express is identical, swapping in `pipeUIMessageStreamToResponse(res)` inside an `app.post` handler; Fastify and Nest.js have cookbook equivalents.)
+
+**MCP client tool usage (stable `@ai-sdk/mcp`):**
+```ts
+import { createMCPClient } from '@ai-sdk/mcp';
+import { Experimental_StdioMCPTransport } from '@ai-sdk/mcp/mcp-stdio';
+import { generateText, stepCountIs } from 'ai';
+
+const client = await createMCPClient({
+  transport: new Experimental_StdioMCPTransport({ command: 'node', args: ['server.js'] }),
+  // or HTTP: transport: { type: 'http', url: 'http://localhost:3000/mcp', headers: {...} }
+});
+try {
+  const tools = await client.tools();
+  const { text } = await generateText({
+    model: 'anthropic/claude-sonnet-4.5',
+    tools,
+    stopWhen: stepCountIs(10),
+    prompt: 'Use the available tools to answer.',
+  });
+  console.log(text);
+} finally {
+  await client.close();
+}
+```
+MCP transports: stdio (local), HTTP (Streamable HTTP), SSE; OAuth supported on HTTP/SSE via `authProvider`. The MCP client (`@ai-sdk/mcp`, ~v1.0.x) is lightweight (tool conversion, resources, prompts, elicitation) but does not yet do session management/resumable streams.
+
+### 5. Ecosystem & Context
+
+- **License:** Apache-2.0 (confirmed on the npm package) — aligns with the user's standards. Fully open source: the ai-sdk.dev homepage states **"12.5M Weekly downloads · 24.8K GitHub stars · 658+ Contributors"** (with ~4.6k forks per the GitHub releases page); ai-sdk.guide reports "over 30 million combined weekly npm installs across the ai core package and @ai-sdk/* providers." Vercel cites 20M+ monthly downloads.
+- **No Vercel lock-in.** The SDK requires no Vercel hosting. Two model-addressing modes: pass a *string* like `'anthropic/claude-opus-4.6'` (routes through the Vercel AI Gateway, needing `AI_GATEWAY_API_KEY`), or pass a *provider instance* like `anthropic('claude-opus-4-6')` that talks **directly** to the vendor with your own key. The Gateway is purely optional convenience: per Vercel's own docs, "AI Gateway reflects provider pricing with no markup and does not charge a platform fee on inference, including on Bring Your Own Key (BYOK) requests." For sovereign deployments, use provider instances or `@ai-sdk/openai-compatible` against self-hosted servers and the Gateway never enters the picture.
+- **vs LangChain.js:** the AI SDK is a focused, strongly-typed TypeScript abstraction over providers + streaming + tools + a lightweight agent loop; LangChain.js offers broader, heavier orchestration abstractions. Downstream agent frameworks (Mastra, Inngest agent kit, even LangChain's TS port) increasingly integrate *against* the AI SDK rather than competing. vs direct provider SDKs: you trade a thin abstraction for provider portability, standardized streaming, and typed tools — generally worth it for multi-provider/at-scale backends.
+- **MCP status:** First-class via `@ai-sdk/mcp` (`createMCPClient`); v6 emphasizes "full MCP support." MCP tools become AI SDK tools transparently.
+- **Crypto / autonomous-agent angle:** The `ToolLoopAgent` running headless in a backend is the natural home for autonomous on-chain agents. **x402** is an HTTP-402 stablecoin micropayment protocol for machine-to-machine payments; the **x402 Foundation launched April 2 2026** under the Linux Foundation (Coinbase contributed the protocol), with named participants including AWS, Google, Visa, Mastercard, Stripe, Circle, Cloudflare, Coinbase, Shopify, Microsoft, Solana Foundation, and Polygon Labs. (Note: Vercel is *not* in the Linux Foundation's named participant list, though it has separately shipped x402 middleware for its serverless functions.) Linux Foundation CEO Jim Zemlin: "The x402 Foundation will create an open, community-governed home to develop these capabilities in the open, ensuring they evolve with transparency, interoperability, and broad participation across the ecosystem." x402 pairs naturally with AI SDK tools — community packages like `x402-agent-tools` expose paid endpoints "as Vercel-compatible tool objects with automatic x402 payment handling," so your agent calls a `tool()`, and the underlying fetch handles the 402 → sign USDC → retry flow without API keys. Self-hosted models via the openai-compatible provider close the sovereignty loop: an agent can run on local inference and pay for external data per request.
+
+## Recommendations
+1. **Start now on v6** with `npm i ai @ai-sdk/anthropic @ai-sdk/openai @ai-sdk/openai-compatible zod`. Pin exact versions for any `experimental_*` API (image/speech/transcription, telemetry, MCP transport). Verify your runtime is Node 18+ (prefer 20/22).
+2. **For sovereignty:** default to provider instances + your own keys, or `@ai-sdk/openai-compatible` against Ollama/vLLM/llama.cpp. Avoid the string-model/Gateway path unless you explicitly want it. Add `extractReasoningMiddleware` for `<think>`-tag local models and `@ai-sdk-tool/parser` middleware for local models lacking native tool calling.
+3. **Structure agents** with `ToolLoopAgent` defined once and reused across CLI/daemon/HTTP. Use `stopWhen` + `prepareStep` for budget/loop safety; set a real step cap (never run unbounded `isLoopFinished()` in production without a token-budget stop condition).
+4. **Observability:** enable `experimental_telemetry` and wire an OTel backend (Langfuse/Logfire/SigNoz). For headless services initialize `@opentelemetry/sdk-node` yourself (the `@vercel/otel` one-liner is Next.js-only).
+5. **Thresholds that change the plan:** if you need resumable/durable agent runs, adopt the `Agent` interface with Workflow DevKit's `DurableAgent` rather than raw `ToolLoopAgent`; if you outgrow in-memory RAG, move `embedMany` output into pgvector/Upstash; if you need many providers with failover and accept the dependency, the Gateway (zero markup, BYOK) becomes worthwhile.
+
+## Caveats
+- **Version drift in third-party content is severe.** Many 2026 blogs still show v4/v5 patterns (`parameters`, `maxSteps`, `Experimental_Agent`, `system` on agents). Always cross-check against ai-sdk.dev (v6) and the GitHub changelog. Some sources speculatively reference "v7 in active beta" and far-future model names (e.g. `claude-opus-4.7`, `gpt-5.4`, `gemini-3-flash`); treat unreleased version/model claims as forward-looking, not confirmed. The exact latest patch differs slightly between sources (GitHub releases `6.0.199`; npm `6.0.202`) because npm may be hours fresher.
+- **Experimental surfaces change without semver guarantees:** image generation, speech, transcription, telemetry, MCP stdio transport, and some agent UI stream helpers are `experimental_`-prefixed. Pin versions.
+- **Streaming reasoning part-type naming varies** between docs (`part.type === 'reasoning'` with `part.textDelta` vs `'reasoning-delta'` with `part.text`); verify against your installed version.
+- **`generateObject`/`streamObject` are deprecated in v6** (still work) — new code should prefer `generateText`/`streamText` with `Output`.
+- Community providers (Ollama, etc.) are "provider dependent" in feature coverage; not all support tools, structured outputs, or multimodal. Verify per model.

docs/actualization_status.md

@@ -0,0 +1,244 @@
+# Actualization status
+
+A per-module map of what's real Python vs. what gracefully degrades to an
+install hint or runtime requirement. Reflects the state after the
+"actualize stubs" pass; counts and labels track the canonical layout from
+[`blueprints/mindxtrain2.md`](blueprints/mindxtrain2.md) §Part 4.
+
+## v1.0.0 production-readiness (objective audit, 2026-06-11)
+
+Honest classification for the v1.0.0 release. **CPU is active**; the GPU path is
+code-complete but needs real ROCm hardware to execute.
+
+**Production-ready on CPU now (run today, no GPU):**
+- Training: `trl_cpu` + `trl_local` (real checkpoints, in-process TRL).
+- Data: `hf`, `local` (JSONL), `mindx_dreams` sources; dedupe/filter/tokenize/pack.
+- Provenance: BLAKE3 manifest + `emit_receipt_for_run` + `verify` + `mindxtrain receipt`.
+- Persona/imprint: script authoring, `imprint` recall before/after, ollama push (LoRA merge).
+- Operator/Coach: recipes, bench dry-run, compile, cost, live-training SSE, receipt card,
+  create-dataset, MEI, training-jobs API, `/v1/chat/completions`.
+- Provenance chain helpers: x402 invoice/settlement, ERC-8004 encode/broadcast (need `--extra chain`).
+
+**GPU-ready, hardware-pending (code complete; needs MI300X/ROCm to run):**
+- Subprocess training backends `axolotl` / `unsloth` / `torchtune` / `primus` (command-built + unit-tested; not executed e2e here).
+- Real autotune probes (attention/GEMM timing) — dry-run reference on CPU.
+- Quark FP8/MXFP4 quantize; vLLM / SGLang serve launchers (commands built, serving needs GPU).
+
+**Stubs — NOT claimed as working in 1.0.0 (roadmap):**
+- `/v1/agentic` mindX MASTERMIND dispatch → `501` (`operator/app.py`).
+- Cloud provisioners `akash` / `ionet` / `bacalhau` / `tensorwave` → `NotImplementedError` (`budget/providers/*`).
+- `lighthouse` as a *data source* and `storage/lighthouse.py:get_dir` → redirect to IPFS.
+
+## Headline numbers
+
+- **99** Python modules under `mindxtrain/`.
+- **38 actualized** (real implementations using stdlib / already-installed deps + lazy imports).
+- **5 cloud-provider stubs** preserved in `budget/providers/*` (post-hackathon).
+- **2 deliberate-redirects** that raise with a pointer to a sibling module
+  (`storage/lighthouse.py:get_dir` → use `storage.ipfs`).
+- **566 tests** pass on a CPU-only laptop (`uv run pytest -q`).
+- **0** OLD-namespace imports anywhere (`from xtrain.`, `from automindx.`,
+  `from custmodel` are all gone).
+
+## What `uv sync` (no extras) gives you
+
+Every module is *importable*. Anything that doesn't need a heavyweight
+runtime works directly:
+
+| Surface | Status |
+|---|---|
+| `mindxtrain --help` / `--version` / `init` / `init --list` | works |
+| `mindxtrain bench --dry-run` | works (synthetic plan) |
+| `mindxtrain receipt <manifest.json>` | works (BLAKE3 verify) |
+| `mindxtrain.operator.app` (FastAPI, no chat backend) | boots; `/coach/` UI live |
+| `mindxtrain.deploy.{registry,hot_swap,ab_test}` | atomic JSON-backed registry |
+| `mindxtrain.operator.{tool_router,agent_loop,context,trajectory,approval}` | bounded ReAct, ContextManager, etc. |
+| `mindxtrain.provenance.{manifest,hashing,verify}` | BLAKE3 manifest round-trip |
+| `mindxtrain.storage.local_fs` | working |
+| `mindxtrain.train.distributed` (FSDP/DeepSpeed config builders) | works |
+| `mindxtrain.budget.{pricing,resource}` | works (psutil if installed) |
+
+## What the optional-dep groups unlock
+
+Install with `uv sync --extra <group>` (multiple `--extra` flags allowed,
+or `--all-extras`):
+
+| Group | Adds | Unlocks |
+|---|---|---|
+| `ml` | `trl`, `transformers`, `peft`, `accelerate`, `datasets` | `mindxtrain train`, `mindxtrain dataset prep`, `mindxtrain.train.{sft,dpo,grpo,rlhf,tool_use}`, `mindxtrain.data.{curate,tokenize}`, `mindxtrain.train.callbacks` |
+| `eval` | `lm-eval`, `lighteval`, `inspect-ai`, `jinja2` | `mindxtrain eval`, `mindxtrain.eval.{harness,lighteval_adapter,inspect_ai_adapter,bfcl,tau_bench,card}` |
+| `data` | `datasketch`, `sentence-transformers`, `faiss-cpu`, `pyarrow` | `mindxtrain.data.{dedupe,filter}` semantic paths, `mindxtrain.eval.persona_regression` |
+| `serve` | `vllm` | in-process vLLM (the operator FastAPI app proxies via httpx by default) |
+| `chain` | `web3`, `py-algorand-sdk`, `huggingface-hub` | `mindxtrain.provenance.{erc8004.broadcast_attestation,x402.validate_settlement,algorand}`, `mindxtrain.storage.hf_hub` |
+| `obs` | `opentelemetry-sdk`, `prometheus-client`, `psutil` | `mindxtrain.operator.telemetry.*`, `mindxtrain.budget.resource.detect` |
+
+The `all` extra installs everything except `amd-quark` (which ships with the
+rocm/primus container — see [HANDOFF.md](HANDOFF.md) §3).
+
+## Per-subpackage status
+
+### `mindxtrain.cli`
+
+`main.py` — **real**. All 9 verbs (`init`, `bench`, `train`, `eval`,
+`quantize`, `serve`, `publish`, `receipt`, `dataset prep`) dispatch into
+canonical modules. Exit codes: `0` = ok, `1` = bad input / missing file,
+`3` = optional dep missing.
+
+### `mindxtrain.config`
+
+`schema.py` (Pydantic 10-section `XTrainConfig`) and `loader.py`
+(YAML render + load) — **real, frozen**. Three runtime-defaults JSON files
+(`train_default.json`, `eval_default.json`, `deploy_default.json`) ship as
+`${ENV}`-interpolated templates per mindxtrain2.md ml-intern style.
+
+### `mindxtrain.data`
+
+| Module | Status | Dep group |
+|---|---|---|
+| `curate.py` | real (HF datasets streaming) | `--extra ml` |
+| `dedupe.py` | real MinHash + SemDeDup | `--extra data` |
+| `filter.py` | real (length/repeat/alpha heuristics + optional KenLM) | none (stdlib) |
+| `pack.py` | real (greedy first-fit + tar shards) | none (stdlib) |
+| `synth.py` | real (httpx → vLLM teacher endpoint) | needs reachable `MINDXTRAIN_TEACHER_BASE_URL` |
+| `tokenize.py` | real (AutoTokenizer wrap) | `--extra ml` |
+| `verify.py` | real (BLAKE3 walk vs manifest) | none |
+
+### `mindxtrain.models`
+
+| Module | Status |
+|---|---|
+| `registry.py` | real (Backend ABC + ModelRegistry + preset registry) |
+| `chat_template.py` | real (Hermes/Qwen3-Coder/Qwen3-Reasoning parsers) |
+| `glm51.py`, `qwen35.py`, `deepseek_v32.py`, `mistral3.py`, `phi4_mini.py` | real Pydantic presets, auto-register on import |
+
+### `mindxtrain.train`
+
+| Module | Status | Dep group |
+|---|---|---|
+| `dispatch.py` | real 4-way switch | none |
+| `axolotl_compile.py` | real (XTrainConfig → Axolotl YAML) | none |
+| `sft.py` | real subprocess wrap of `accelerate launch -m axolotl.cli.train` | `--extra ml` + axolotl on PATH |
+| `dpo.py`, `grpo.py`, `rlhf.py`, `tool_use.py` | real TRL trainer wraps | `--extra ml` |
+| `distributed.py` | real (FSDP / DeepSpeed dict builders, 1- or 8-GPU only) | none |
+| `callbacks.py` | real `EvalDuringTraining` + `BestCheckpointKeeper` | `--extra ml` (lazy) |
+| `backend_unsloth.py`, `backend_torchtune.py`, `backend_primus.py` | real subprocess wraps | each backend's own install |
+
+### `mindxtrain.eval`
+
+| Module | Status | Dep group |
+|---|---|---|
+| `harness.py` | real `lm_eval` subprocess + JSON parser | `--extra eval` |
+| `lighteval_adapter.py` | real `lighteval accelerate` wrap | `--extra eval` |
+| `inspect_ai_adapter.py` | real `inspect eval` wrap | `--extra eval` |
+| `bfcl.py` | real `bfcl evaluate` wrap | external (BFCL harness) |
+| `tau_bench.py` | real subprocess wrap | external |
+| `persona_regression.py` | real (sentence-transformer cosine vs baseline) | `--extra data` |
+| `agenda_regression.py` | real (keyword overlap + optional LLM judge) | none + optional `MINDXTRAIN_TEACHER_BASE_URL` |
+| `card.py` | real (Jinja2 with stdlib `string.Template` fallback) | optional `--extra eval` |
+
+### `mindxtrain.autotune`
+
+| Module | Status |
+|---|---|
+| `benchmark.py`, `plan.py`, `gemm_probe.py`, `rccl_probe.py` | real |
+| `attention_probe.py` | real (CK vs Triton SDPA timing if torch+ROCm available; CPU fallback returns canonical default) |
+
+### `mindxtrain.operator`
+
+| Module | Status |
+|---|---|
+| `app.py` (FastAPI), `coach/api.py`, `coach/static/*` | real |
+| `tool_router.py` | real (typed `ToolSpec` + dispatch) |
+| `agent_loop.py` | real (bounded ReAct + doom-loop detector) |
+| `context.py` | real (170k-token compaction + summarize fallback) |
+| `trajectory.py` | real (JSONL append-only writer) |
+| `approval.py` | real (CLI / Web / Slack transports) |
+| `backends/{vllm,openai_compat}.py` | real (httpx clients to OpenAI-compat endpoints) |
+| `telemetry/{energy,otel_hooks,prometheus_exporter}.py` | real, gracefully no-op if optional deps missing |
+| `prompts/{system_v1,codephreak}.yaml` | real prompt-as-data |
+
+### `mindxtrain.storage`
+
+| Module | Status | Dep group |
+|---|---|---|
+| `provider.py` | real ABC | none |
+| `local_fs.py` | real | none |
+| `hf_hub.py` | real (huggingface_hub upload_folder) | `--extra chain` |
+| `lighthouse.py` | real httpx POST to Lighthouse REST API; falls back to stub-CID without `LIGHTHOUSE_API_KEY` | none |
+| `ipfs.py` | real httpx to kubo `/api/v0/add` | needs running kubo |
+
+### `mindxtrain.provenance`
+
+| Module | Status | Dep group |
+|---|---|---|
+| `manifest.py` | real (`Manifest` + `emit_receipt`) | none |
+| `hashing.py` | real (BLAKE3 file/dir) | none |
+| `verify.py` | real (re-hash on-disk artifacts) | none |
+| `x402.py` | real httpx invoice + Algorand verify | `--extra chain` |
+| `erc8004.py` | real ABI encode + web3 broadcast | `--extra chain` |
+| `algorand.py` | real BANKON ENS allocator + ASA info | `--extra chain` |
+
+### `mindxtrain.deploy`
+
+| Module | Status | Dep group |
+|---|---|---|
+| `registry.py` | real atomic JSON-backed registry | none |
+| `hot_swap.py` | real canary-promote + rollback | none |
+| `ab_test.py` | real deterministic Splitter | none |
+| `api_client.py` | real httpx → mindx.pythai.net + agenticplace.pythai.net | needs deployed services |
+| `vllm_launcher.py`, `sglang_rocm.py` | real argv builders | none |
+| `quark.py` | real subprocess wrap of `python -m amd_quark.quantize` | rocm/primus container |
+| `gptq_rocm.py` | real subprocess wrap | `auto-gptq` ROCm wheel |
+
+### `mindxtrain.budget`
+
+| Module | Status | Dep group |
+|---|---|---|
+| `pricing.py` | real | none |
+| `resource.py` | real (psutil + rocm-smi probes; falls back to defaults) | optional `--extra obs` |
+| `providers/{akash,amd_dev_cloud,bacalhau,ionet,tensorwave}.py` | **stubs** (post-hackathon) | each provider's SDK |
+
+## What stays as `NotImplementedError`
+
+7 residual `NotImplementedError` raises across the package:
+
+- `budget/providers/akash.py`, `amd_dev_cloud.py`, `bacalhau.py`, `ionet.py`,
+  `tensorwave.py` — cloud-burst provisioning. Out of hackathon scope.
+- `storage/lighthouse.py:LighthouseProvider.get_dir` — deliberately
+  redirects to `mindxtrain.storage.ipfs.IpfsProvider.get_dir`.
+- `train/dispatch.py` — string match in a docstring, not an actual raise.
+
+Run `grep -r "raise NotImplementedError" mindxtrain` to confirm.
+
+## Test coverage
+
+```
+tests/
+├── test_ab_test.py                   # canary splitter distribution
+├── test_agent_loop.py                # bounded ReAct + doom-loop
+├── test_autotune_plan.py             # AutotunePlan invariants
+├── test_axolotl_compile.py           # XTrainConfig → Axolotl YAML
+├── test_cli_smoke.py                 # all 9 verbs reachable
+├── test_coach_api.py                 # /coach/api/* endpoints
+├── test_config_schema.py             # 10-section schema, recipe round-trip
+├── test_context_manager.py           # ContextManager compaction
+├── test_data_pipeline.py             # filter / synth / verify
+├── test_deploy_registry.py           # registry + hot-swap atomicity
+├── test_distributed.py               # FSDP/DeepSpeed builders, xGMI invariant
+├── test_manifest.py                  # Manifest + BLAKE3 round-trip
+├── test_models_registry.py           # preset + chat-template lookup
+├── test_pack.py                      # greedy first-fit packer + tar shards
+├── test_parsers.py                   # chat templates
+├── test_pricing.py                   # MI300X $/hr math
+├── test_provenance_verify.py         # tamper detection
+├── test_tool_router.py               # ToolSpec dispatch
+└── test_vllm_launcher.py             # vLLM cmd builder
+```
+
+`uv run pytest -q` → **564 passed**.
+
+## See also
+
+- [HANDOFF.md](HANDOFF.md) — ordered checklist for taking the project from "code is done" to "demo is live."
+- [development.md](development.md) — toolchain, lazy-import pattern, how to add features.
+- [architecture.md](architecture.md) — canonical layout + 5-layer architecture.

docs/architecture.md

@@ -0,0 +1,170 @@
+# Architecture
+
+`mindxtrain` is a single-package training framework producing checkpoints with
+verifiable provenance, served through an OpenAI-compatible API. The repository
+is organized per `docs/blueprints/mindxtrain2.md` §Part 4.
+
+```
+mindxtrain/
+├── cli/         entry point (typer): init|bench|train|eval|quantize|serve|publish|receipt
+├── config/      Pydantic schema + JSON / YAML loaders
+├── data/        curate -> dedupe -> filter -> tokenize -> pack -> synth -> verify
+├── models/      ModelRegistry + ChatTemplate + per-base presets
+├── train/       sft, dpo, grpo, rlhf, tool_use, distributed, callbacks, recipes/*.yaml
+├── eval/        lighteval, inspect_ai, bfcl, persona/agenda regression, tau_bench, card
+├── autotune/    60-second AOT MI300X probe (the differentiator)
+├── operator/    FastAPI app, Coach UI, ml-intern patterns (tool_router, agent_loop, …)
+├── storage/     StorageProvider interface + local_fs / hf_hub / lighthouse / ipfs
+├── provenance/  TrainingRun manifest, BLAKE3, ERC-8004, Algorand, x402
+├── deploy/      content-addressed registry, hot_swap, ab_test, vllm/sglang launchers, quark
+└── budget/      psutil-derived ResourceBudget + per-provider pricing
+```
+
+## The five conceptual layers
+
+The codebase is concentric — each inner layer is consumed by the next, never
+the reverse.
+
+```
+┌──────────────────────────────────────────────────────────┐
+│ 1. CLI layer (typer)                                     │
+│    init | bench | train | dataset prep | eval | quantize │
+│    serve | publish | receipt                             │
+├──────────────────────────────────────────────────────────┤
+│ 2. Autotune layer (60s AOT probe — DIFFERENTIATOR)       │
+│    attention_probe (CK vs Triton) │ gemm_probe │ rccl    │
+│      ↓                                                   │
+│    AutotunePlan (JSON, AOT — JIT autotune is forbidden)  │
+├──────────────────────────────────────────────────────────┤
+│ 3. Dataset layer                                         │
+│    HF datasets streaming → MinHash + SemDeDup → packing  │
+│    → FSDP sharding → Lighthouse-pinned CIDs              │
+├──────────────────────────────────────────────────────────┤
+│ 4. Training layer (backend dispatch)                     │
+│    axolotl │ unsloth │ torchtune │ primus                │
+│    LoRA │ QLoRA │ full SFT │ DPO │ ORPO │ GRPO │ GSPO    │
+├──────────────────────────────────────────────────────────┤
+│ 5. Artifact + Integration layer                          │
+│    Quark FP8 / MXFP4 → lm-eval-harness                   │
+│    → HF Hub push → Lighthouse pin                        │
+│    → mindX register → AgenticPlace listing               │
+│    → BANKON ENS subname → x402 Algorand metering         │
+└──────────────────────────────────────────────────────────┘
+```
+
+The CLI never reaches into the training backend; it consumes the autotune plan
+and a Pydantic-validated config and dispatches downward through
+`mindxtrain/train/dispatch.py`. The training backend never reaches up to the
+CLI; it returns a checkpoint directory that the artifact layer consumes.
+
+## Autotune is the spine
+
+The single architectural choice that distinguishes mindxtrain from Axolotl,
+LLaMA-Factory, Unsloth, torchtune and Primus is the autotune layer. It runs a
+**60-second MI300X micro-benchmark** (CK-vs-Triton SDPA, hipBLASLt heuristic
+check, RCCL bus-bandwidth probe) and emits a static `AutotunePlan` JSON
+consumed at training start.
+
+**AOT-only — JIT autotune is forbidden in production.** The plan is fixed at
+training start; no Triton / Inductor / MIOpen JIT autotune runs in the live
+training loop. This is reproducible, latency-stable, and the point of the
+entire framework.
+
+See [autotune.md](autotune.md) for the full probe taxonomy and the
+`AutotunePlan` schema.
+
+## MI300X-specific invariants (non-negotiable)
+
+These are encoded in the schema and the recipe library; violating them is a
+deployment bug.
+
+1. **FSDP topology must be 1- or 8-GPU** (`hardware.gpus: Literal[1, 8]`). The
+   2- and 4-GPU groups have asymmetric xGMI bandwidth on MI300X — kills
+   throughput silently. Enforced by the schema; tested in
+   `tests/test_config_schema.py`.
+2. **`PYTORCH_ROCM_ARCH=gfx942`** must be set; AOTriton compiles for the GPU
+   arch and `gfx942` is MI300X. Default in every recipe's `train.env`.
+3. **`HSA_NO_SCRATCH_RECLAIM=1`** + **`HIP_FORCE_DEV_KERNARG=1`** +
+   **`GPU_MAX_HW_QUEUES=1`** — the three runtime knobs that make Primus-Turbo
+   MI300X paths stable. Default in every recipe's `train.env`.
+4. **Numpy must be pinned `<2.0`** against `torch==2.9.1+rocm7.2.1.lw`. Pinned
+   in the project `pyproject.toml`.
+5. **Container is `rocm/primus:v26.2`**; SHA256 digest snapshot lives in
+   `ops/containerfiles/digest.lock`.
+
+## End-to-end data flow
+
+```
+examples/demo_qwen3_8b_sft.yaml
+        │
+        ├─[parse, validate]─► XTrainConfig (Pydantic v2)
+        │
+        ├─[mindxtrain bench]─► AutotunePlan {ck/triton, gemm, rccl, …}
+        │                           │
+        │                           ▼
+        ├─[mindxtrain train]──► dispatch_training(cfg, plan, out_dir)
+        │                           │
+        │                           ▼ (Axolotl YAML, env vars set)
+        │                       checkpoint_dir/  + train.log
+        │                           │
+        ├─[mindxtrain eval]─────►   eval.json (lm-eval-harness)
+        │                           │
+        ├─[mindxtrain quantize]─►   checkpoint_dir/quantized/  (Quark FP8 PTPC)
+        │                           │
+        └─[mindxtrain publish]──►   Manifest with BLAKE3 of YAML+dataset+
+                                    checkpoint+eval, plus HF/Lighthouse/
+                                    INFT/ASA pointers
+                                                 │
+                                                 ▼
+                                    mindxtrain.operator.app serves the FP8
+                                    weights on /v1/chat/completions
+```
+
+`mindxtrain receipt` re-hashes the artifacts and verifies the BLAKE3 fields
+against the manifest. That round-trip is the cypherpunk2048 reproducibility
+guarantee.
+
+## Model strategy (per mindxtrain2.md §Part 6)
+
+mindxtrain targets a **family**, not a single flagship: edge → mid →
+flagship → specialist. Per the rigorous comparison in
+[`blueprints/mindXtrain2.md`](blueprints/mindXtrain2.md) §Part 6:
+
+- **Primary base = Qwen3.5** (Apache-2.0, contiguous family from 0.6 B →
+  235 B → Qwen3.5-122B-A10B; mature PEFT/Axolotl/Unsloth recipes; BFCL
+  leadership in the Qwen lineage).
+- **Specialist track = GLM-5.1** (MIT-licensed weights; SOTA SWE-Bench Pro
+  58.4; long-horizon agentic reasoning with 200 K-context DSA). Used where
+  8-hour autonomous SWE sessions matter; otherwise overkill.
+- DeepSeek V3.2 / Mistral Large 3 / Phi-4-mini / Gemma 4 are watchlist or
+  jurisdictional secondary tracks.
+
+The five `mindxtrain.models.{glm51,qwen35,deepseek_v32,mistral3,phi4_mini}.py`
+preset modules auto-register on import so `mindxtrain init` and the
+`ModelRegistry` know all five from day one.
+
+## Actualization status
+
+The framework ships with **38 modules actualized** as real Python on a
+CPU-only laptop, **6 optional-dep groups** (`ml`, `eval`, `data`, `serve`,
+`chain`, `obs`) for the heavyweight paths, and **5 cloud-provider stubs**
+preserved in `budget/providers/*` for future work. Per-module map at
+[actualization_status.md](actualization_status.md).
+
+Verification gate that should always pass on the base install:
+
+```bash
+uv run pytest -q          # → 564 passed
+uv run ruff check .       # clean
+```
+
+## What lives outside the Python tree
+
+- `contracts/` — Foundry workspace for `mindxtrain_registry.sol` (write-once
+  anchor) and `x402_receiver.sol` (immutable facilitator). No proxies, no
+  admin keys, no setters.
+- `examples/` — `demo_qwen3_8b_sft.yaml` (the hero config).
+- `Containerfile`, `compose.yaml` — top-level Podman / podman-compose entries.
+- `ops/` — per-role container files, compose stacks, k8s manifests, vmm and
+  Gensyn definitions.
+- `docs/blueprints/` — the source design briefs the project was built against.

docs/autotune.md

@@ -0,0 +1,153 @@
+# Autotune — the 60-second AOT probe
+
+The single layer that distinguishes mindxtrain from Axolotl, LLaMA-Factory, Unsloth, torchtune, and Primus. Quoted from the frozen design brief in `docs/blueprints/`:
+
+> The single most differentiating angle is the auto-selection layer. No competitor framework — not Axolotl, LLaMA-Factory, Unsloth, torchtune, or Optimum-AMD itself — runs a per-job MI300X micro-benchmark before training to pick CK vs Triton attention backends, hipBLASLt heuristic vs rocBLAS path, AITER vs reference MoE kernels, NCCL_MIN_NCHANNELS, gradient-checkpointing strategy, FSDP shard width, and LoRA rank against the actual (model, dataset shape, sequence length, GPU count) tuple. mindxtrain owns that AOT-only autotune layer.
+
+## The AOT-only discipline
+
+JIT autotune (Triton autotune in vLLM cold-start, `torch.compile(mode='max-autotune')` Inductor, MIOpen find-mode) is **forbidden in production training**. Reasons:
+
+1. **Reproducibility.** A run with JIT autotune produces different kernels on different invocations of the same workload, breaking deterministic benchmarks.
+2. **First-batch latency.** Triton autotune on cold start can stall a training step for 5-30 seconds, invisible in the loss curve and very visible in `tok/s`.
+3. **Cypherpunk2048 standard.** Production paths must be statically declared at deployment. JIT compilation is an in-band runtime decision, which is exactly what the standard prohibits.
+
+The `autotune.policy: aot_only` field in the YAML is the contract. The training layer reads the `AutotunePlan` JSON at start, sets env vars + flags, and never re-tunes during the loop. AOTriton (the AOT version of Triton math) is loaded as a precompiled `.so`; Composable Kernel kernels are pulled from the offline-tuned hipBLASLt cache.
+
+## The probe taxonomy
+
+`mindxtrain bench` runs three probes in sequence inside its 60-second budget. The whole flow is at [`mindxtrain/autotune/benchmark.py`](../mindxtrain/autotune/benchmark.py).
+
+### 1. attention_probe — CK vs Triton SDPA
+
+[`mindxtrain/autotune/attention_probe.py`](../mindxtrain/autotune/attention_probe.py).
+
+Times `torch.nn.functional.scaled_dot_product_attention` across four representative shapes (queries × keys × heads × head-dim per the recipe's `model.name` + `data.seq_len`) on both backends:
+
+| Backend  | How                                                                |
+|----------|--------------------------------------------------------------------|
+| `ck`     | Composable Kernel (default) — hand-tuned ASM/CK kernels via AITER.  |
+| `triton` | AOTriton 0.11.2b0 with `TORCH_BLAS_PREFER_HIPBLASLT=0` and `PYTORCH_TUNABLEOP_ENABLED=0` toggles. |
+
+The probe is **real** — when `torch` (`--extra ml`) and a ROCm-visible GPU
+are both present, it times the four representative shapes on each backend
+via `torch.nn.attention.sdpa_kernel`. Without torch (typical CPU dev box),
+the probe gracefully returns the canonical `("ck", [])` default so
+`bench --dry-run` parity holds and the AutotunePlan downstream consumers
+keep working unchanged.
+
+```
+budget: ~30 s
+shapes: 4 representative (qlen, klen, num_heads, head_dim)
+output: AttentionBackend ∈ {ck, triton}, list[ProbeTiming]
+```
+
+`ProbeTiming` is `{ label, backend, median_ms, iterations }` — captured per (shape × backend) so the demo can render a side-by-side timing table in the video.
+
+### 2. gemm_probe — hipBLASLt heuristic
+
+[`mindxtrain/autotune/gemm_probe.py`](../mindxtrain/autotune/gemm_probe.py).
+
+Per the user-confirmed Day-1 plan ("1 real probe + 2 hardcoded heuristics"), this returns `hipblaslt_default` for gfx942 based on AMD's documented MI300X tuning guidance. Reference: AMD ROCm 7.2.1 release notes, hipBLASLt 0.10 default heuristics are within 5 % of hand-tuned for the BF16/FP16 GEMMs mindxtrain hits (LoRA rank 16-64, hidden 2048-8192).
+
+**Why we don't enumerate.** A real hipBLASLt heuristic enumeration is ~1.5 minutes and risks burning the entire 60-second budget. If MMLU eval shows GEMM-bound throughput regression on a specific recipe, revisit later.
+
+Output: `hipblaslt_default | hipblaslt_tuned | rocblas_fallback`.
+
+### 3. rccl_probe — collective bandwidth
+
+[`mindxtrain/autotune/rccl_probe.py`](../mindxtrain/autotune/rccl_probe.py).
+
+For 1-GPU runs this is a no-op. For 8-GPU runs it returns `8gpu_xgmi` with `NCCL_MIN_NCHANNELS=112` set in the plan notes. **2-GPU and 4-GPU groupings raise `RuntimeError`** — MI300X xGMI bandwidth between subsets of 2/4 GPUs is asymmetric, and FSDP shards on those topologies will silently bottleneck.
+
+```python
+def probe_rccl(gpu_index: int = 0, gpu_count: int = 1) -> RcclConfig:
+    if gpu_count == 1:
+        return "1gpu_noop"
+    if gpu_count == 8:
+        return "8gpu_xgmi"
+    raise RuntimeError(f"FSDP on {gpu_count} GPUs is unsafe...")
+```
+
+This is enforced in two places: the `rccl_probe` raises at probe time, and the `XTrainConfig.hardware.gpus` field is `Literal[1, 8]` so the schema rejects bad values at parse time.
+
+## The `AutotunePlan` schema
+
+[`mindxtrain/autotune/plan.py`](../mindxtrain/autotune/plan.py).
+
+```python
+class AutotunePlan(BaseModel):
+    schema_version: Literal["1"] = "1"
+    gpu_arch: str = "gfx942"
+    rocm_version: str = "7.2.1"
+
+    attention_backend: Literal["ck", "triton"] = "ck"
+    gemm_heuristic: Literal["hipblaslt_default", "hipblaslt_tuned", "rocblas_fallback"] = "hipblaslt_default"
+    rccl_config: Literal["1gpu_noop", "8gpu_xgmi", "unsupported_2_4_gpu"] = "1gpu_noop"
+
+    fsdp_shard_width: Literal[1, 8] = 1
+    suggested_lora_rank: int = 16
+    suggested_micro_batch_size: int = 4
+
+    probe_timings: list[ProbeTiming] = []
+    notes: list[str] = []
+```
+
+Pure data, content-addressed via BLAKE3 in the mindxtrain provenance manifest, fully reproducible across MI300X nodes.
+
+## How the training layer consumes the plan
+
+`mindxtrain/train/dispatch.py` reads the plan and applies it before invoking
+the backend (real subprocess wrap of `accelerate launch -m axolotl.cli.train`
+in `mindxtrain/train/sft.py`):
+
+```python
+def dispatch_training(cfg: XTrainConfig, plan: AutotunePlan, out_dir: Path) -> Path:
+    # 1. set env vars: cfg.train.env + plan-driven additions
+    #    e.g. plan.rccl_config == "8gpu_xgmi" → set NCCL_MIN_NCHANNELS=112
+    #         plan.attention_backend == "ck" → NVTE_CK_USES_BWD_V3=1, etc.
+    # 2. compile cfg → Axolotl YAML, override:
+    #    train.flash_attention.backend ← plan.attention_backend
+    #    train.method.r ← plan.suggested_lora_rank if cfg.train.method.kind == "lora"
+    #    train.batch.per_device ← min(cfg, plan.suggested_micro_batch_size)
+    # 3. subprocess: accelerate launch -m axolotl.cli.train <yaml>
+    # 4. capture stdout/stderr to out_dir/train.log
+    # 5. return checkpoint dir
+```
+
+## Dry-run / CI path
+
+Every CI pipeline runs `mindxtrain bench --dry-run`, which skips the GPU probes entirely and emits a hardcoded reference plan. The reference plan has `attention_backend: ck`, `gemm_heuristic: hipblaslt_default`, `rccl_config: 1gpu_noop`, `fsdp_shard_width: 1` — sane MI300X 1-GPU defaults that exercise the same code path the real probe writes.
+
+```bash
+$ uv run mindxtrain bench --dry-run --out plan.json
+wrote plan.json (dry_run=True, attention=ck, gemm=hipblaslt_default)
+```
+
+The dry-run path is what makes the GitHub Actions CI matrix CPU-only.
+
+## Day 2 implementation budget (target ~30 minutes per probe)
+
+| Probe          | Target time | Risk                                              |
+|----------------|-------------|---------------------------------------------------|
+| attention_probe | 30 min     | First-time AOTriton compilation may be slow; warm cache mounted from persistent volume. |
+| gemm_probe     | 0 (hardcoded) | None — heuristic is documented.                |
+| rccl_probe     | 5 min       | 1-GPU is no-op; 8-GPU only relevant if we rent the 8× SKU. |
+
+Total Day 2 budget: ~35 minutes of MI300X time + writing time. The remaining hours go to verifying the plan flows into Axolotl correctly via Day 3's dispatch wiring.
+
+## Where the demo wow-moment lives
+
+Capture `autotune_plan.json` and the streaming probe output for the 5-minute video. The 60-second autotune dashboard is the single most quotable visual asset in the submission — a measurable kernel selection that no competitor framework ships.
+
+```
+$ mindxtrain bench --gpu 0 --out plan.json
+[autotune] CK FA forward, shape=(8, 4096, 32, 128): 12.4 ms (median over 50)
+[autotune] Triton FA forward, shape=(8, 4096, 32, 128): 14.7 ms (median over 50)
+[autotune] CK FA forward, shape=(8, 4096, 16, 128): 11.0 ms
+[autotune] Triton FA forward, shape=(8, 4096, 16, 128): 13.2 ms
+[autotune] picked ck (avg 1.2× faster)
+[autotune] gemm: hipblaslt_default (gfx942 documented heuristic)
+[autotune] rccl: 1gpu_noop
+[autotune] wrote plan.json (1.2 KB) in 47 s
+```

docs/benchmarks.md

@@ -0,0 +1,69 @@
+# Benchmarks
+
+The numbers we are chasing on the hero workload, and the framework comparison that goes in the README.
+
+## Hero workload
+
+**Qwen3-8B SFT, 1× MI300X, bs=8, seq=4096, BF16, AdamW, 1B tokens.**
+
+| Metric                   | Target          | Why                                                            |
+|--------------------------|-----------------|----------------------------------------------------------------|
+| Throughput               | **>15 000 tok/s** | Comparable to AMD's published Llama-3.1-8B numbers.           |
+| MFU                      | **>40 %**         | Floor for "MI300X is being exercised, not idled."              |
+| Time to eval-loss = 1.5  | **<90 minutes**   | Lets the demo video show the loop converging in real time.     |
+| Total cost               | **<$3**           | $1.99 / hr × 1 GPU × ~1.5 hr × safety margin.                  |
+| Peak HBM                 | ~80 GB          | Headroom on MI300X's 192 GB; impossible on H100 80 GB.         |
+
+Hit those four and the cost slide writes itself: **MI300X $1.99/hr × 1 GPU × 1.5 hr ≈ $3** versus **H100 $4/hr × 2 GPUs × 4 hr ≈ $32** — 4× cheaper for the same workload, and the H100 baseline can't even fit BF16 at this batch/seq combo without quantization.
+
+## H100 cost baseline
+
+The argument the judges remember is "MI300X is 4× cheaper for this exact workload." The cost numbers come from public list prices and need to hold up under questioning.
+
+| GPU       | $/hr  | Memory  | Qwen3-8B BF16 bs=8 seq=4096 | Cost for 1B tokens |
+|-----------|-------|---------|-----------------------------|--------------------|
+| H100 80 GB | $4.00 | 80 GB  | OOM unless bs/seq cut       | ~$32 (2× GPUs, 4 hr, FP8 fallback) |
+| H200 141 GB | $6.00 | 141 GB | Fits, ~12k tok/s             | ~$24 (1× GPU, 4 hr) |
+| MI300X 192 GB | $1.99 | 192 GB | Fits with headroom, >15k tok/s | **<$3** (1× GPU, 1.5 hr) |
+
+## Framework comparison (the README differentiator)
+
+This is the table the README prints. mindxtrain is the only row with all seven cells filled — that is the elevator pitch.
+
+| Framework      | One-cmd ROCm 7.2.1 install | MI300X auto-tune | Qwen3.6 day-zero | FP8 via Quark | x402 micropayments | Decentralized fallback | Training-receipt manifest |
+|----------------|----------------------------|------------------|------------------|---------------|--------------------|------------------------|---------------------------|
+| Axolotl        | ⚠ (community fork)        | ✗                | ✓                | △ (torchao)  | ✗                  | ✗                      | ✗                         |
+| LLaMA-Factory  | ✓ (AMD tutorial)           | ✗                | ✓                | △            | ✗                  | ✗                      | ✗                         |
+| Unsloth        | ✓ (OneClickAMD)            | ✗                | △ (single-GPU)   | ✗            | ✗                  | ✗                      | ✗                         |
+| torchtune      | ✓ (AMD CI)                 | ✗                | ✗ (no recipe)    | △            | ✗                  | ✗                      | ✗                         |
+| Primus         | ✓ (`rocm/primus:v26.2`)    | ✗                | ✗ (pretrain only) | ✓           | ✗                  | ✗                      | ✗                         |
+| **mindxtrain** | **✓**                      | **✓ (60s AOT)**  | **✓**            | **✓**         | **✓ (Algorand)**   | **✓ (Bacalhau/Akash)** | **✓ (BLAKE3 + INFT)**     |
+
+Legend: ✓ = supported · ⚠ = supported via community fork · △ = partial / opt-in · ✗ = not supported.
+
+## Capturing the numbers
+
+The `mindxtrain` CLI emits structured logs that map onto the metrics above. The output tree:
+
+```
+runs/<run_id>/
+├── config.yaml              # input, BLAKE3-hashed in the manifest
+├── autotune_plan.json       # the AOT plan (the differentiator)
+├── train.log                # accelerate stdout/stderr
+├── metrics.jsonl            # one record per logging step: tok_per_s, mfu, hbm_gb, watts
+├── checkpoint/              # HF safetensors + tokenizer, BLAKE3-hashed
+├── quantized/               # Quark FP8 PTPC, vLLM-loadable
+├── eval.json                # lm-evaluation-harness output
+└── manifest.json            # mindxtrain.provenance.Manifest with BLAKE3 hashes
+```
+
+`metrics.jsonl` is the source of truth for the benchmark numbers. The cost slide is a one-liner over that file: average `tok_per_s` × seconds × $1.99 / 3600.
+
+## Regression detection
+
+`eval.regression.threshold_pct: -1.0` in every recipe means **fail the run if any benchmark task drops more than 1 percentage point** versus the base model baseline. That keeps a fine-tune that improves the target distribution but breaks general capability from being silently published. The baseline JSON is computed once per base model and cached alongside the run; comparison happens via `mindxtrain.eval.persona_regression.regression_score` and `mindxtrain.eval.agenda_regression.regression_score`.
+
+## What's not measured (yet)
+
+- Energy (kWh per training run) — `mindxtrain.operator.telemetry.energy.sample_power_w` wraps `rocm-smi --showpower` (returns 0.0 W gracefully on a CPU dev box). MI300X power baseline is ~750 W under load; a 90-minute run is ~1.1 kWh. Telemetry collection into `metrics.jsonl` is wired but the dashboard integration is post-hackathon work.
+- Multi-node throughput — out of hackathon scope; the `mindxtrain receipt` manifest accommodates it (`hardware.gpus` field), and the autotune `rccl_probe` is the entry point for the multi-node version.

docs/blueprints/Winning the AMD x lablab.ai Developer Hackathon with mindX and xtrain_ A Three-Track Strategic Brief.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0afa520dabeeaf21bd547366e66e851d330211c0c884151730c12333281de83
+size 820481

docs/blueprints/mindXtrain Framework_ GLM-5.1, aGLM Lineage, and Qwen3.5 Primary Base Strategy.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe2c3db125a2b1b33f6c50ddcd2918f33ee13138d896c7a844950345bdf2f372
+size 1188768

docs/blueprints/mindXtrain.md

@@ -0,0 +1,119 @@
+# Winning all three tracks of the AMD x lablab.ai Developer Hackathon with mindX
+
+**codephreak — this is your operating brief.** The AMD Developer Hackathon hosted by lablab.ai is a **7-day online build (May 4–10, 2026)** with an **invitation-only on-site finale May 9–10, 2026** at the **MindsDB SF AI Collective, 3154 17th St, San Francisco**. The total prize pool is **$21,500+ plus one AMD Radeon AI PRO R9700 GPU**, with **$100 of AMD Developer Cloud (DigitalOcean-hosted MI300X) credits** per registered AMD AI Developer Program member — roughly 50 GPU-hours on a 192 GB MI300X at the published $1.99/hr rate. **Registration deadline was May 3, 2026**; submissions are due by end of May 10. The event is structured as **three primary tracks plus a meta "Build in Public" challenge and an optional cross-track x402 Payments / "Launch & Fund Your Startup" challenge** — meaning a single, well-architected mindX submission can legitimately compete in three primary tracks and two side-pool challenges concurrently. The structural answer to your question is: **no rule prohibits a single project from being entered into all three primary tracks** (lablab's submission form requires you to select "Main Tracks" — plural — and the AMD hackathon's track copy explicitly invites cross-track work via the Build-in-Public and x402 challenges). What the rules *do* require is that the project demonstrate meaningful, load-bearing use of MI300X-class hardware in each track it claims — generic "Llama-on-AMD" wrappers will lose. Your existing mindX/AgenticPlace/BANKON stack maps with unusual cleanliness onto every track, and the xtrain module you intend to build is the missing piece that converts mindX from a cognitive-API/agent-marketplace play into a credible AMD-native training-and-inference platform — which is exactly the "build across the AI stack" thesis the AMD blog announcing this hackathon explicitly states.
+
+This brief is exhaustive. It documents the event end-to-end, the AMD developer stack at the canonical-URL level, the architecture for an integrated three-track submission, the xtrain module design, the day-by-day execution plan for the remaining ~6 days, and a verbatim link inventory at the end.
+
+## The hackathon as it actually exists
+
+Despite a few stale third-party summaries that report **only $10,000 and only "May 9–10"** (CompeteHub, the original lablabai X post), the canonical lablab page and the AMD launch blog both state **$21,500+ total** and a **May 4–10** online build window, with the SF on-site weekend as a culmination, not the entire event. The discrepancy is a real-world signal: the prize pool was upgraded mid-flight when Hugging Face, Akash Systems, MindsDB, NYSE Wired, theCUBE, and Qwen joined as partners, and lablab pushed the "Hugging Face Community joining" upgrade through Facebook and X. The AMD Developer Hackathon's official Hugging Face organization at **huggingface.co/lablab-ai-amd-developer-hackathon** is where live submissions accumulate as Spaces and models — by capture, **224 team members had joined and were already shipping projects** like SentinelBrain-14B-MoE (training live on MI300X), MediAgent (5-agent medical pipeline), REPOMIND (256K-context coding agent on a single MI300X), BrainConnect-ASD, AndesOps-AI, and Paperhawk. The competitive field is real and fast.
+
+The **three primary tracks** are: **AI Agents & Agentic Workflows** (positioned as "best track for beginners," tech stack LangChain/CrewAI/AutoGen against open-source models served via vLLM endpoints — Llama, DeepSeek, Mistral, Qwen); **Fine-Tuning on AMD GPUs** (advanced/GPU-intensive, ROCm + PyTorch + Hugging Face Optimum-AMD + vLLM, targeting domain LLMs in healthcare/finance/legal/code on MI300X); and **Vision & Multimodal AI** (high-throughput multimodal apps — Llama 3.2 Vision, Qwen-VL — exploiting MI300X's 192 GB VRAM and 5.3 TB/s HBM3 bandwidth to run full-precision rather than quantized). The **Build in Public** track is a parallel, cross-track meta prize requiring three or more technical posts on X or LinkedIn tagged **#AMDDevHackathon**, plus open-sourcing the project or publishing a technical walkthrough, plus submitting structured ROCm/Dev Cloud feedback. The **x402 Payments / "Launch & Fund Your Startup"** challenge is the side-pool challenge that runs explicitly **alongside any hackathon track**, asking for an AI-native product with **X402 programmable payments** demonstrating either an agent-to-agent autonomous payment loop or a built-in revenue model (token-gated access, real-time rev-splits, instant payouts). Special prizes layered on top include a **Hugging Face Spaces "Most Likes"** prize (Reachy Mini Wireless + 6 months HF PRO + $500 HF credits for 1st), a **Social Engagement** GPU prize, and a **Best Overall** project prize.
+
+The **judging criteria** are the four equally-weighted lablab standards used at every recent event: **Application of Technology** (how meaningfully MI300X / ROCm / Dev Cloud are integrated — not "API wrapped"), **Presentation** (deck + video clarity), **Business Value** (practical impact, real business areas), and **Originality** (creative angle). lablab's submission rules are unambiguous: a **working live demo URL the judges can test in real time** is required, the **video presentation is capped at 5 minutes** and uploaded as a link with the file under 300 MB, the long description must be at least 100 words, the cover image is 16:9, and the canonical license expectation is **MIT-compliant open source unless track says otherwise** — which conflicts with your cypherpunk2048 Apache-2.0 standard and must be reconciled (a permissive **dual-license note in the README is acceptable** in practice; many lablab winners ship Apache-2.0 with an explicit MIT-compatibility statement). Submissions are made through the lablab.ai project form on the hackathon page; the form fields are Submission Title (≤50 chars), Short Description (≤255 chars), Long Description (≥100 words), Main Tracks, Technologies, Cover Image, Video Presentation URL, Demo Application URL, and Additional Information (where the scaling/business plan lives).
+
+The **schedule** layered onto the on-site weekend, per the Luma RSVP page, includes a project submission workshop and pitching-form explanation at **11:10 AM** by Joanna Słupczewska of lablab.ai. Named on-site speakers/judges are **Pawel Czech** (CEO NativelyAI, founder of lablab.ai) and **Ramine Rozen** (Corporate VP, AI at AMD). The recurring lablab head judge across recent events is **Walaa Nasr Elghitany** (PhD, PMP). Recurring lablab mentors observed on adjacent events — **Paulo Almeida, Theodoros Ampas, Shebagi Mitra, Donald Nwokoro, Iqra Akhtar, Dimitrije Pešić, Muhammad Inaamullah** — should be expected to mentor here as well. Community channels are the **lablab.ai Discord** (~64,110 members) and the **AMD Developer Discord** (separate, AMD-run); the Hugging Face hackathon org is the live submission-staging surface. There is also a parallel **AMD x GPU MODE E2E Kernel Speedrun** with a separate $1.1 M prize purse — not the lablab event, but commonly conflated.
+
+## The AMD developer stack you will actually use
+
+The compute substrate is the **AMD Instinct MI300X** (CDNA 3, gfx942, 304 CUs, 192 GB HBM3, 5.325 TB/s, 1,307 TFLOPS BF16/FP16 dense, 2,615 TFLOPS FP8), with optional **MI325X** (same compute, 256 GB HBM3E, 6.0 TB/s) on later DigitalOcean SKUs. CDNA 4's **MI350X / MI355X** (288 GB HBM3E, 8 TB/s, MXFP4/MXFP6 native) is GA, but the AMD Developer Cloud tier exposed for this hackathon is MI300X. The driver stack as of May 2026 is the **ROCm 7.2.x production stream** (current patch **7.2.2**, GA from Jan 2026 CES under 7.2.0; HIP/CLR 7.2.53211, AMD Clang 22.0.0, MIOpen 3.5.1, MIGraphX 2.15.0, RCCL 2.27.7, Composable Kernel 1.2.0, AOTriton 0.11.2b0, Triton 3.5.1/3.6.0, hipBLASLt 1.2.2). A second "TheRock" preview stream (7.12.0) exists but is **explicitly not for production**; pin to 7.2.2. Canonical entry points are **rocm.docs.amd.com** (with the compatibility matrix at `/en/latest/compatibility/compatibility-matrix.html` as the single source of truth) and **github.com/ROCm/ROCm**. Note that several historically separate repos — MIOpen, RCCL, rccl-tests, Composable Kernel, the BLAS/SPARSE/RAND families — have been consolidated under **github.com/ROCm/rocm-libraries** and **github.com/ROCm/rocm-systems**; legacy URLs still resolve but PRs route to the monorepos.
+
+For training, the canonical container is **rocm/pytorch:rocm7.2.1_ubuntu24.04_py3.12_pytorch_release_2.9.1** on Docker Hub, run with `--cap-add=SYS_PTRACE --security-opt seccomp=unconfined --device=/dev/kfd --device=/dev/dri --group-add video --ipc=host --shm-size 8G`. PyTorch 2.9.1, 2.8.0, and 2.7.1 are all supported on ROCm 7.2; the install index for nightlies is `https://download.pytorch.org/whl/nightly/rocm7.2`. **PyTorch FSDP and FSDP2 are first-class on MI300X**, and the AMD-validated path for serious distributed work is the **AMD-AIG-AIMA/torchtitan-amd** fork on the `dev/primus_turbo` branch, orchestrated by **AMD-AGI/Primus** (container `rocm/primus:v26.2`), with the **Primus-Turbo** operator library providing FlashAttention, GroupedGEMM, AITER, CK, hipBLASLt, and Triton kernels — the only AMD-side path with FP8 mixed-precision training that works (Transformer Engine is NVIDIA-only). Critical MI300X runtime knobs are `TORCH_NCCL_HIGH_PRIORITY=1`, `GPU_MAX_HW_QUEUES=2`, `PYTORCH_TUNABLEOP_ENABLED=1`, and `PYTORCH_ROCM_ARCH=gfx942`. **A non-obvious gotcha that has eaten dozens of teams**: **avoid 2-GPU and 4-GPU collective groups on MI300X** — xGMI bandwidth between subsets of 2/4 GPUs is asymmetric, so design FSDP shards to be either 1-GPU or full 8-GPU. RCCL **2.27.7** has a fixed allreduce data-corruption bug for sub-512 KiB messages on MI350X/MI355X; harmless on MI300X but verify if you migrate.
+
+For inference, **vLLM is the reference engine**, with images at **rocm/vllm** (production) and **rocm/vllm-dev** (weekly), constraint pin `vllm>=0.17.0,<0.19.0` paired with `aotriton==0.11.2b0`, `amd-aiter==0.1.10.post2`, `triton==3.6.0`, `torch==2.10.0`, `xformers==0.0.34`. **AITER** (AI Tensor Engine for ROCm, github.com/ROCm/aiter) is the default kernel backend on AMD for LLM inference and provides hand-tuned ASM/CK/Triton kernels for FlashAttention, GEMM, fused MoE, MLA decode, FP8/FP4 quantization, and two-shot allreduce; it is integrated into vLLM, SGLang, ATOM, and Primus-Turbo. The non-vLLM serving alternative AMD pushes is **SGLang** with the **Mooncake** distributed-KV-cache plugin, used by the AMD/Xiaomi MiMo-V2.5-Pro deployment playbook. For graph-level inference compilation, **MIGraphX 2.15.0** (github.com/ROCm/AMDMIGraphX, ONNX-Runtime EP) is preferred over the deprecated ROCm-EP. For quantization, **AMD Quark 0.11.1** (`pip install amd-quark`, docs at quark.docs.amd.com) handles PTQ/QAT across int4/8/16, FP8 (E4M3/E5M2), MXFP4/MXFP6, GPTQ, AWQ, SmoothQuant, and Qronos, and produces vLLM-loadable models. **Pre-quantized AMD models** like `amd/Llama-2-70b-chat-hf-WMXFP4FP8-AMXFP4FP8-AMP-KVFP8` are at huggingface.co/amd. AMD's own open models that judges respond to — and that are perfect for fine-tune demos — are **Instella-3B-Instruct** (3 B params, 128 K context variant available, MI300X-trained), **Instella-VL-1B** (vision-language), **AMD-OLMo-1B**, **AMD-Llama-135m / -135m-code**, and the **Nitro** diffusion family (Nitro-1, Nitro-T-0.6B/1.2B, Nitro-E). The **AMD GAIA** open-source agent framework (github.com/amd/gaia, MIT-licensed, current v0.17.0) is the canonical AMD agent reference and pairs naturally with the AI Agents track. **Ryzen AI / XDNA 2 / GAIA / Lemonade SDK** are not relevant here because the hackathon's compute is cloud MI300X, not Strix Halo client devices — though Lemonade is a useful reference for serving abstractions.
+
+Sign-up flow: enroll in the **AMD AI Developer Program** at amd.com/en/developer/ai-dev-program.html, then provision your VM at **devcloud.amd.com** (1× MI300X for ~$1.99/hr, 8× MI300X for ~$15.92/hr); the program's **$100 hackathon credit** translates to ~50 hours of single-MI300X. Quick-Start images preloaded with ROCm + PyTorch + vLLM + JupyterLab eliminate dependency drift. Pre-arm one writable volume for **MIOpen kernel cache** (`~/.cache/miopen/<ver>`), **AITER JIT cache** (`AITER_JIT_DIR`), and **Torch extensions** (`TORCH_EXTENSIONS_DIR`) — without these, first-iteration latency on every restart will burn your demo window.
+
+## How mindX, AgenticPlace, and BANKON map onto each track
+
+The thesis you are pitching is that **mindX is the cognitive AI brain, AgenticPlace is the marketplace where mindX-trained agents become rentable, and BANKON is the identity/payment plane via x402-on-Algorand and ENS subnames** — and that **the AMD Developer Hackathon's three tracks plus the x402 side challenge plus the Build-in-Public meta track are the four faces of one coherent system**. This is the "build across the AI stack" framing the AMD launch blog explicitly invited. The integrated submission's name should be something like **"mindX + xtrain on AMD: a sovereign cognitive-training and agent-marketplace stack with x402-Algorand metering."** Demo URL: **mindx.pythai.net** with a hackathon-specific landing route (`mindx.pythai.net/hackathon`) wiring the three demos behind a single judge-friendly tab UI.
+
+For the **AI Agents & Agentic Workflows track**, mindX is already a multi-agent control framework built around MASTERMIND (orchestrator), automindx (cognitive runtime), Ollama-driven self-improvement readiness, and the SocraticReasoning / SimpleCoder agents documented in your existing rage.pythai.net architecture. The MI300X-load-bearing argument is: a 70B-class model (Llama 3.3 70B, Qwen3-Coder-70B, or DeepSeek-V3-Lite) running in **full BF16 on a single MI300X via vLLM** is mindX's "MASTERMIND.consciousness" reasoning core, with sub-agents (`SimpleCoder`, `MediAgent-style`, `LogicTables`, RAGE retrieval) coordinated through your existing automindx orchestrator. This is exactly the architecture pattern judges have been rewarding (REPOMIND used 256K context on a single MI300X; MediAgent uses a 5-agent medical pipeline). Your differentiator is **agent-as-marketplace-listing**: every mindX agent registers itself on **agenticplace.pythai.net**, gets a BANKON-managed ENS subname (e.g., `mediagent.bankon.eth`), and exposes its endpoints behind an x402 paywall so a calling agent autonomously pays per inference call. **Concrete deliverables**: a live `mindx.pythai.net/hackathon/agents` page where judges can paste a query, see the MASTERMIND graph route across three to five mindX agents on MI300X, see the x402 invoice and Algorand settlement, and see the called agents' AgenticPlace listings update with real on-chain usage stats.
+
+For the **Fine-Tuning on AMD GPUs track**, this is where **xtrain** earns its place. The framing: mindX has historically been an inference-and-orchestration layer; xtrain is the new training subsystem that lets mindX self-improve and also lets third parties fine-tune domain models inside the AgenticPlace marketplace, with each training job priced and metered via x402-Algorand. The MI300X-load-bearing argument is **single-GPU full-parameter LoRA fine-tuning of a 70B model in BF16 with no quantization compromise**, or **QLoRA of a 405B model**, both impossible at full precision on 80 GB H100s. Use the **AMD AI Academy "GRPO on a single MI300X" workflow** as your reference (it is explicitly cited in the hackathon's own zero-to-builder article). **Concrete deliverable**: a live job at `mindx.pythai.net/hackathon/xtrain` that takes a Hugging Face dataset ID and a base model (default `amd/Instella-3B-Instruct` so judges see an AMD model being improved on AMD hardware), tokenizes via HuggingFace `datasets`, runs LoRA via PEFT + PyTorch FSDP2 on MI300X with Primus-Turbo BF16, evaluates against `lm-evaluation-harness` and a custom benchmark, persists checkpoints to **Lighthouse/IPFS**, mints an **ERC-7857 INFT** on Algorand-bridged Base (or your chosen mainnet) recording the model artifact's content hash and rights, and lists the resulting LoRA adapter as a rentable mindX agent on AgenticPlace. The training job itself is metered: x402 payment from caller wallet → Algorand settlement → MI300X allocation → checkpoint URI returned. **Use AMD Quark to FP8-quantize the final LoRA-merged model** so the same artifact serves on vLLM in the agent track, completing the train→serve→sell loop.
+
+For the **Vision & Multimodal AI track**, lean on **Instella-VL-1B** or **Qwen3-VL-4B** at full precision on MI300X with long-image-context retrieval through RAGE. The pitch is a **multimodal cognitive analyst** built into mindX: feed it a PDF, a slide deck, a chart screenshot, or an X-ray; mindX routes via RAGE to a vision sub-agent on MI300X, fuses the structured output into the MASTERMIND reasoning graph, and returns a long-form structured analysis. Two clean demo verticals to pick: **drAIML medical** (your existing healthcare consultant identity) doing radiology-style multimodal triage, or **codephreak codebase analyzer** doing whole-repo visual+code understanding (architecture diagrams + source files). The MI300X-load-bearing argument is full-resolution unquantized vision with 70B-class language fusion in a single GPU. **Concrete deliverable**: `mindx.pythai.net/hackathon/multimodal` with a drag-and-drop input, an MI300X-side latency counter, and AgenticPlace listings showing a "drAIML Visual" agent rentable per call.
+
+For the **x402 Payments side challenge**, this is your structural advantage — the **parsec-wallet x402-Algorand** layer in BANKON is already production-grade. Wire xtrain training jobs, AgenticPlace agent invocations, and the Hugging Face Spaces demo all behind x402 invoices that settle on Algorand in seconds. Build one specifically scored deliverable: a **two-agent autonomous payment loop** (the lablab x402 challenge's "Agent-to-Agent" sub-challenge) where mindX's RAGE retriever calls a third-party data API priced in USDC-on-Algorand via x402, with the agent dynamically choosing whether to pay based on a confidence threshold. This satisfies the x402 challenge's first option directly, and the "built-in revenue model" option through the xtrain job-metering and AgenticPlace listings simultaneously.
+
+For the **Build-in-Public meta track**, you are already operationally a writer (rage.pythai.net is a living archive). Commit to **at least five technical posts** between now and submission, each tagged **#AMDDevHackathon @AIatAMD @lablabai**: (1) a "why MI300X for sovereign cognition" framing post; (2) an xtrain architecture deep-dive with FSDP2 and Primus-Turbo notes; (3) a benchmark post comparing your LoRA pipeline vs an H100 cost baseline; (4) an x402-Algorand training-job-metering walkthrough with on-chain receipts; (5) a recap of the integrated three-track demo. Pair these with a **structured ROCm/Dev Cloud feedback document** delivered through the lablab submission form's feedback field — the Build-in-Public reward weights detailed feedback heavily. Open-source the complete mindX repo with a top-level `HACKATHON.md` linking each post.
+
+**Risk factors and disqualifiers**: lablab requires an **MIT-compliant** submission. Your cypherpunk2048 standard is Apache-2.0; the resolution is to ship **Apache-2.0 with an explicit "MIT-compatible" notice** in the `LICENSE-NOTICE.md` and SPDX identifier `Apache-2.0` plus `LICENSE-MIT-COMPAT.md` mirroring permissions — verify with the lablab Discord #ineedhelp channel before submission to be safe. Lablab also expects the **demo URL to be live during judging**; budget MI300X uptime for the full judging window (typical pattern is 48–72 hours after submission close). The "all three tracks" claim must be substantiated with **three distinct working demos under one umbrella**, each individually testable; do not rely on a single combined demo where judges have to imagine the track-specific functionality. Track judges are different per track; speak in the language of each track in your README sub-sections.
+
+**Whether one project can win all three primary tracks**: the rules do not forbid it, and the submission form's "Main Tracks" field is plural. However, the realistic competitive analysis is that **dedicated specialists win specialist tracks**. Your structurally optimal play is: **submit one integrated project to all three primary tracks plus the x402 challenge plus Build-in-Public, and win on the combined "Best Overall" prize and at least two of the three primary tracks**. Winning the third primary track depends on whether a dedicated specialist outscores you on Application of Technology in their narrow lane. The "Best Overall" prize is the asymmetric upside — a full-stack play wins it almost by definition over single-track entries.
+
+## xtrain module design
+
+xtrain is the cleanest single-week deliverable that converts mindX from a cognitive layer into a production training-and-marketplace platform, and it is what makes the fine-tuning track winnable rather than a low-effort PEFT script. Its public name in the repo: `mindx/xtrain/` (flat snake_case per cypherpunk2048). License: Apache-2.0 with MIT-compatibility notice. Python ≥3.12. Container: Podman with a ROCm 7.2.1 base.
+
+**Architecture in flowing detail**: xtrain is a Python package with a CLI (`xtrain run --config job.yaml`) and a FastAPI service (`xtrain serve`) exposing job-submission, status, and artifact-retrieval endpoints behind x402 invoices. A submitted job is a YAML manifest specifying a base model (HF ID), a dataset (HF ID or IPFS CID), a training recipe (LoRA, QLoRA, or full SFT), a hardware target (`mi300x_1`, `mi300x_8`, `mi325x_1`), and a budget cap in USDC. The orchestrator validates the manifest, computes a deterministic content-hash job ID, issues an **x402 invoice** through parsec-wallet, waits for Algorand settlement, then provisions an AMD Developer Cloud droplet (or attaches to an existing reserved one) via the DigitalOcean REST API, transfers the dataset (sharded via HF `datasets` `streaming=True` to avoid storing 100 GB+ corpora locally), tokenizes inside the container, and launches the training run.
+
+The **training engine** is **PyTorch 2.9.1 + Primus + torchtitan-amd** for full SFT and 70B-class LoRA, and **PEFT + Hugging Face TRL + Unsloth-on-ROCm** for the lightweight LoRA/QLoRA path that maps onto the AMD AI Academy GRPO recipe. **DeepSpeed-ROCm** is wired in as a fallback for ZeRO-3 + CPU offload jobs that exceed even MI300X memory, with `DS_BUILD_SPARSE_ATTN=0 DS_BUILD_EVOFORMER_ATTN=0` to avoid the unsupported ops. The shard topology is constrained to **1-GPU or 8-GPU FSDP groups** to dodge the MI300X 2/4-GPU xGMI degradation. FlashAttention v3 via AOTriton, AITER fused MoE, hipBLASLt for GEMMs, and Primus-Turbo's FP8 mixed-precision are all enabled by default for 70B-class jobs. Training is BF16-default with FP8 opt-in. **Determinism**: seed everything, pin Triton/AOTriton commits, and persist the **MIOpen kernel cache** to a Lighthouse-pinned blob keyed by `(rocm_version, gfx_arch, model_arch)` so subsequent jobs warm-start without JIT compilation cost.
+
+**AOT-only artifact policy compliance** per cypherpunk2048: xtrain emits **only AOT-compiled artifacts** for production serving — no JIT torch.compile in the deployed inference path. The training side intentionally JIT-compiles (Triton, AITER, MIOpen kernels) because that's where AOTriton's name comes from — it is **ahead-of-time-emitted Triton math at deploy time**, despite the training-time JIT compilation. The serving artifacts written by xtrain are: (1) merged HF-format checkpoint, (2) **AMD Quark FP8 quantized variant** (`pip install amd-quark`, MXFP4 for MI350X if the cloud SKU upgrades), (3) AOTriton-emitted SDPA kernels precompiled for gfx942, (4) MIGraphX-compiled ONNX graph for the inference path that doesn't go through vLLM, and (5) a deployment manifest pointing vLLM at the FP8 weights with the correct `--dtype auto` and AITER env vars. The serving container then runs zero JIT — purely AOT artifacts loaded from disk. This is the cypherpunk2048 standard verbatim, applied correctly to the ROCm reality.
+
+**Data pipeline**: HF `datasets` (`load_dataset(..., streaming=True)`) for ingestion, `tokenizers` for tokenization with the base model's tokenizer (cached per-model in IPFS), then **Sharded WebDataset** (`.tar` shards) emitted to a Lighthouse-pinned bucket with content-hash keys; the training run reads via `webdataset` with `nodesplitter=ResampledShards` so the 8-GPU job gets balanced shards. For instruction-tuning datasets, default to the `amd/Instella-GSM8K-synthetic` style format; provide `--format alpaca|sharegpt|openai_messages|custom_jinja` switches. **Eval datasets** are similarly streamed; the eval harness wraps `lm-evaluation-harness` (MMLU, ARC, HellaSwag, GSM8K, HumanEval) and **MTEB** for embedding models, plus a **custom benchmark** registered through a `xtrain.eval.register_benchmark` decorator so AgenticPlace marketplace listings can advertise per-task scores.
+
+**Checkpoint persistence**: every checkpoint is written to **Lighthouse** (Filecoin-pinned IPFS, `lighthouse.storage` SDK) with a deterministic CID. The CID, training config, eval scores, dataset hash, and base-model hash are committed to an on-chain **ERC-7857 INFT** record. ERC-7857 (the Intelligent NFT standard for tokenized AI assets) lets you tokenize the model artifact with on-chain rights metadata — the perfect vehicle for AgenticPlace's "rent or buy a model" UX. Mint on Base mainnet (cheap, EVM-compatible, x402-friendly via Coinbase facilitator) and bridge metadata to Algorand for the x402 payment side. Use **Foundry** as the canonical Solidity test framework: `forge test` for unit tests on the INFT factory, `forge script` for deploy, with the `lib/` git-submodule pattern. The Foundry test suite covers minting, transfer, royalty splits to the model creator, and the rights-revocation path.
+
+**Hyperparameter search**: integrate **Optuna** for sequential search and **Ray Tune on ROCm** for parallel search across 8 MI300X GPUs. Optuna's TPE sampler is the default; Ray Tune is opt-in via `--search ray-asha`. Pruners cut unproductive trials early. Each trial's intermediate metrics are streamed back to the xtrain orchestrator and visible in the live mindX dashboard at `mindx.pythai.net/hackathon/xtrain/runs/<job_id>` — judges can watch a hyperparameter sweep happen in real time.
+
+**Model registry tied to ERC-7857 INFT**: a `xtrain.registry` module reads the on-chain INFT registry, hydrates model metadata (CID, eval scores, lineage to base model), and exposes a `xtrain.registry.list()` and `xtrain.registry.fetch(cid)` API. AgenticPlace queries this registry to render the marketplace UI; mindX queries it to load weights at inference time. The lineage graph (model A fine-tuned from model B fine-tuned from model C) is stored as an on-chain merkle DAG with each node's INFT pointing to its parent, so royalties cascade.
+
+**x402 Algorand payment metering for compute-as-a-service**: the xtrain FastAPI service issues HTTP 402 responses with x402-spec headers when a job is submitted without payment proof. The caller (a parsec-wallet, an AgenticPlace agent, or a third-party CLI) signs an x402 invoice referencing an Algorand asset (USDC ASA), the facilitator settles in sub-second finality, and the xtrain server validates the on-chain proof before launching the job. Pricing function: per-GPU-hour rate × estimated_steps × safety_margin, refundable via post-job true-up against actual compute consumed. The Algorand settlement is the primary surface; Base settlement via the Coinbase x402 facilitator is the alternate path for callers preferring EVM rails. The same metering wraps **inference** calls when xtrain-trained models are served on AgenticPlace — every model invocation is an x402 transaction, completing your monetization loop.
+
+**AgenticPlace integration**: trained models surface as `/agents/<cid>` listings with metadata, eval scores, pricing, and a "Try" button that issues an x402-paid call against a vLLM endpoint. The `chainmapping` module from **agenticplace.pythai.net/allchain.html** exposes the multi-chain deployment registry — Base, Algorand, ENS subname, and any other chains your BANKON identity layer covers — so an AgenticPlace listing carries the complete cross-chain identity of the model and its creator. Foundry tests exercise the INFT contract on Base; pytest exercises the off-chain Python layer; an end-to-end integration test runs `forge script` to deploy to a local Anvil fork, mints an INFT, then pytest-drives a full xtrain LoRA on a tiny model and verifies the resulting CID lands in the registry.
+
+**Mainnet deployment path**: on submission day, xtrain's INFT factory deploys to **Base mainnet** with the Algorand x402 facilitator pointed at the **Algorand mainnet USDC ASA**. Demo wallets are pre-funded with $5 USDC on each chain so judges can run a paid xtrain job end-to-end without leaving the demo URL. The INFT contract address and the Algorand application ID are published in the submission's README and embedded in the demo UI as copy-pasteable links to the block explorers (basescan.org and allo.info / pera explorer).
+
+## Day-by-day execution plan (May 4 — May 10)
+
+You have lost zero days if you start tonight. The realistic plan acknowledges that the on-site SF weekend is invitation-only and the online submission deadline closes EOD May 10. Here is the day map.
+
+**Day 1 (May 4, today)**: Sign up for AMD AI Developer Program; provision a **single MI300X droplet** at devcloud.amd.com using your $100 credit; pull `rocm/pytorch:rocm7.2.1_ubuntu24.04_py3.12_pytorch_release_2.9.1`, `rocm/vllm:latest`, and `rocm/primus:v26.2`; verify `rocminfo` reports gfx942 and `nvidia-smi`-equivalent `rocm-smi` shows the full 192 GB. Spin up a persistent volume mount for `~/.cache/miopen`, `AITER_JIT_DIR`, and `TORCH_EXTENSIONS_DIR`. Clone mindX, AgenticPlace, and BANKON repos onto the droplet. Register on lablab.ai for the hackathon (deadline was May 3 but late enrollment is sometimes permitted via Discord ping — message the lablab #ineedhelp channel immediately). Join the AMD Developer Discord and the lablab Discord. Ship Build-in-Public post #1: "Why MI300X is the right substrate for sovereign cognition."
+
+**Day 2 (May 5)**: Stand up the **integrated demo skeleton at `mindx.pythai.net/hackathon`** with three sub-routes (`/agents`, `/xtrain`, `/multimodal`) and a fourth shared route (`/x402`) for the payment loop demo. Wire the existing mindX MASTERMIND orchestrator to a vLLM backend serving Llama 3.3 70B in BF16 on the MI300X (`docker run rocm/vllm:latest --model meta-llama/Llama-3.3-70B-Instruct --dtype bfloat16 --tensor-parallel-size 1` — this fits with headroom on 192 GB). Verify token throughput with `vllm bench`. Start the AgenticPlace marketplace pointed at the new MI300X endpoint. Ship Build-in-Public post #2: "Running Llama 3.3 70B unquantized on a single GPU — what 192 GB unlocks."
+
+**Day 3 (May 6)**: Build the **xtrain module's first vertical slice**. Implement the FastAPI service skeleton, the YAML job manifest schema, the x402 invoice issuance, and a working LoRA fine-tune of `amd/Instella-3B-Instruct` on a small Alpaca-format dataset with HF PEFT + TRL on the MI300X. Use `bf16=True`, `gradient_checkpointing=True`, `fsdp="full_shard auto_wrap"`. Persist the resulting LoRA adapter to Lighthouse and capture the CID. Mint a placeholder ERC-7857 INFT on Base Sepolia with `forge create`. Eval the adapter against `lm-evaluation-harness` MMLU/HellaSwag and capture the scores in the registry. Ship Build-in-Public post #3 with code snippets and timing.
+
+**Day 4 (May 7)**: **Multimodal track day**. Spin up a second vLLM container serving **Instella-VL-1B** or **Qwen3-VL-4B** at full precision on a second GPU partition (or share the GPU via vLLM's `--gpu-memory-utilization 0.4` and run both 70B and the VLM concurrently — 192 GB makes this trivially possible). Build the drag-and-drop multimodal demo at `/multimodal` with a drAIML medical imaging vertical. Wire it into the MASTERMIND graph so the multimodal agent is callable from the agent track demo too. Ship Build-in-Public post #4: "Multimodal at full precision: drAIML medical triage on MI300X."
+
+**Day 5 (May 8)**: **x402 + AgenticPlace integration day**. Wire the parsec-wallet x402-Algorand client into mindX so every AgenticPlace agent invocation issues an x402 invoice; demonstrate the autonomous agent-to-agent payment loop where mindX's RAGE retriever pays a third-party API agent. Move the INFT factory from Base Sepolia to **Base mainnet**. Pre-fund demo wallets with USDC on Algorand and Base. Run end-to-end: judge clicks "Train", x402 invoice issues, judge's demo wallet pays, xtrain provisions the MI300X (or attaches to your existing droplet), training runs, INFT mints, AgenticPlace listing appears. Foundry tests pass: `forge test --gas-report`. Ship Build-in-Public post #5: "x402 on Algorand, settling AI training jobs in 2 seconds."
+
+**Day 6 (May 9)**: **Polish, video, deck day**. Record the **5-minute demo video**: 30 seconds of mindX/AgenticPlace/BANKON framing, 60 seconds of the Agents track demo (judge query → MASTERMIND graph → 70B response), 60 seconds of the xtrain fine-tune demo (job submission → MI300X training → INFT mint → marketplace listing), 60 seconds of the multimodal demo, 60 seconds of the x402 payment loop, 30 seconds of close. 16:9, under 300 MB, hosted on YouTube unlisted with the URL ready. Build the **pitch deck** with the lablab-recommended structure: problem, mechanics, tech, user case study (screen recording inset). Write the **README and HACKATHON.md** with the four judging criteria as named sub-sections (Application of Technology, Presentation, Business Value, Originality), the link to each Build-in-Public post, the structured ROCm/Dev Cloud feedback, and the license notice. Run a full end-to-end rehearsal twice. If you receive a SF on-site invite, fly out the morning of the 9th; otherwise demo from your existing setup.
+
+**Day 7 (May 10)**: **Submit early in the day**, not at the deadline. lablab's submission form sometimes degrades under EOD load. Submit through `lablab.ai/ai-hackathons/amd-developer` with all three primary tracks selected, the x402 challenge box checked, the Build-in-Public box checked, and Long Description ≥100 words emphasizing the integrated three-track architecture. After submission, **keep the demo URL live for 72 hours** for judging access — do not tear down the MI300X droplet. Ship Build-in-Public post #6 (recap), tagging @AIatAMD and @lablabai, and submit the structured ROCm feedback form.
+
+## GitHub repo structure following cypherpunk2048
+
+The repo is `github.com/Professor-Codephreak/mindx-xtrain` (or wired into `pythaiml/mindx`). Top-level: `LICENSE` (Apache-2.0), `LICENSE-NOTICE.md` (MIT-compatibility statement for lablab judging), `README.md`, `HACKATHON.md` (pointing to demo URL, video, posts, deck, on-chain addresses), `pyproject.toml` (Python ≥3.12, hatchling backend), `Containerfile` (Podman, ROCm 7.2.1 base), `compose.yaml` (Podman-compose for the full stack: vLLM + xtrain server + AgenticPlace + Algorand sandbox + Anvil fork), `foundry.toml`, `lib/` (Forge submodules: `forge-std`, `openzeppelin-contracts`, `solady`), `src/` (Solidity: `XTrainINFTFactory.sol`, `XTrainRegistry.sol`, `XTrainRoyaltySplit.sol`), `test/` (Foundry: `XTrainINFT.t.sol` etc.), `script/` (Foundry deploy: `Deploy.s.sol` Base mainnet target), `mindx/` (Python: `xtrain/`, `agents/`, `mastermind/`, `rage/`), `tests/` (pytest), `docs/` (architecture diagrams, API references), `.github/workflows/ci.yml` (Foundry tests + pytest + ruff + mypy on PR). Flat snake_case throughout `mindx/`. No JIT torch.compile in any production path — every serving entrypoint loads AOT artifacts. SPDX headers on every Solidity file. README opens with the BLUF demo URL, video URL, and three-track pitch in five sentences.
+
+## Complete link inventory
+
+**Primary hackathon page and direct AMD/lablab URLs**: https://lablab.ai/ai-hackathons/amd-developer ; https://www.amd.com/en/developer/resources/technical-articles/2026/build-across-the-ai-stack--join-the-amd-x-lablab-ai-hackathon-.html ; https://lablab.ai/ai-articles/from-zero-to-ai-builder-amd-developer-program ; https://luma.com/afz0aeq8 ; https://huggingface.co/lablab-ai-amd-developer-hackathon ; https://www.competehub.dev/en/competitions/lumaac244e2451ac6091f3c1a1ff6bc04b0d ; https://foundersbay.com/events/lablab-amd-developer-hack ; https://x.com/lablabai/status/2037263372014514272 ; https://www.facebook.com/lablabai/photos/the-amd-developer-hackathon-just-got-a-major-upgrade-huggingfacecommunity-is-joi/1001171305991438/ ; https://lablab.ai/ai-hackathons ; https://lablab.ai/ ; https://lablab.ai/event ; https://lablab.ai/guide ; https://lablab.ai/blog/hackathon-guidelines ; https://lablab.ai/ai-articles/hackathon-guidelines ; https://lablab.ai/hackathon-rules ; https://lablab.ai/blog/guidelines-for-creating-a-project-pitch ; https://lablab.ai/delivering-your-hackathon-solution ; https://lablab.ai/tech ; https://lablab.ai/apps/recent-winners ; https://lablab.ai/ai-tutorials/x402-ai-payments-hackathon-tutorial ; https://lablab.ai/tech/coinbase/x402 ; https://discord.com/invite/lablabai ; https://discord.gg/XnxrJ8ytRs ; https://discord.com/invite/amd-dev ; https://www.amd.com/en/developer/ai-dev-program.html ; https://developer.amd.com/events/ ; https://www.amd.com/en/corporate/events/amd-ai-dev-day.html .
+
+**AMD Developer Cloud and access**: https://devcloud.amd.com ; https://www.amd.com/en/developer/resources/cloud-access/amd-developer-cloud.html ; https://www.amd.com/en/developer/resources/cloud-access.html ; https://www.amd.com/en/developer/resources/technical-articles/2025/how-to-get-started-on-the-amd-developer-cloud-.html ; https://www.amd.com/en/developer.html ; https://www.amd.com/en/blogs/2025/introducing-the-amd-developer-cloud.html ; https://www.amd.com/en/blogs/2025/enabling-the-future-of-ai-introducing-amd-rocm-7-and-the-amd-developer-cloud.html ; https://www.amd.com/en/blogs/2025/100k-hours-free-developer-cloud-access.html ; mailto:devcloudrequests@amd.com .
+
+**Instinct hardware product pages**: https://www.amd.com/en/products/accelerators/instinct/mi300/mi300x.html ; https://www.amd.com/en/products/accelerators/instinct/mi300/platform.html ; https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/data-sheets/amd-instinct-mi300x-data-sheet.pdf ; https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/data-sheets/amd-instinct-mi300x-platform-data-sheet.pdf ; https://www.amd.com/en/products/accelerators/instinct/mi300.html ; https://www.amd.com/en/products/accelerators/instinct/mi300/mi325x.html ; https://www.amd.com/en/products/accelerators/instinct/mi300/mi325x/platform.html ; https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/product-briefs/instinct-mi325x-datasheet.pdf ; https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/product-briefs/instinct-mi325x-platform-datasheet.pdf ; https://www.amd.com/en/products/accelerators/instinct/mi350.html ; https://www.amd.com/en/products/accelerators/instinct/mi350/mi350x.html ; https://www.amd.com/en/products/accelerators/instinct/mi350/mi355x.html ; https://www.amd.com/en/blogs/2025/amd-instinct-mi350-series-and-beyond-accelerating-the-future-of-ai-and-hpc.html ; https://www.amd.com/en/blogs/2025/amd-instinct-mi350-series-game-changer.html .
+
+**ROCm core docs and meta-distribution**: https://rocm.docs.amd.com/ ; https://rocm.docs.amd.com/en/latest/ ; https://rocm.docs.amd.com/en/latest/about/release-notes.html ; https://rocm.docs.amd.com/en/latest/compatibility/compatibility-matrix.html ; https://rocm.docs.amd.com/projects/install-on-linux/en/latest/ ; https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html ; https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/3rd-party/pytorch-install.html ; https://rocm.docs.amd.com/en/latest/compatibility/ml-compatibility/pytorch-compatibility.html ; https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference-optimization/workload.html ; https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference/benchmark-docker/vllm.html ; https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference/benchmark-docker/previous-versions/vllm-history.html ; https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/training/benchmark-docker/primus-pytorch.html ; https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/training/benchmark-docker/primus-megatron.html ; https://rocm.docs.amd.com/projects/ai-developer-hub/en/latest/ ; https://rocm.docs.amd.com/projects/ai-developer-hub/en/latest/notebooks/pretrain/torch_fsdp.html ; https://rocm.docs.amd.com/projects/ai-developer-hub/en/latest/notebooks/pretrain/torchtitan_llama3.html ; https://rocm.docs.amd.com/projects/ai-developer-hub/en/latest/notebooks/pretrain/torchtitan_deepseek.html ; https://rocm.docs.amd.com/projects/ai-developer-hub/en/latest/notebooks/gpu_dev_optimize/aiter_mla_decode_kernel.html ; https://rocm.blogs.amd.com/ ; https://rocm.blogs.amd.com/artificial-intelligence/fsdp-training-pytorch/README.html ; https://rocm.blogs.amd.com/artificial-intelligence/quark/README.html ; https://rocm.blogs.amd.com/software-tools-optimization/aiter-ai-tensor-engine/README.html .
+
+**ROCm GitHub repos**: https://github.com/ROCm/ROCm ; https://github.com/ROCm/TheRock ; https://github.com/ROCm/HIP ; https://github.com/ROCm/clr ; https://github.com/ROCm/HIPIFY ; https://github.com/ROCm/hipify_torch ; https://github.com/ROCm/AMDMIGraphX ; https://github.com/ROCm/torch_migraphx ; https://github.com/ROCm/MIOpen ; https://github.com/ROCm/rocm-libraries ; https://github.com/ROCm/rocm-systems ; https://github.com/ROCm/rccl ; https://github.com/ROCm/rccl-tests ; https://github.com/ROCm/aws-ofi-rccl ; https://github.com/ROCm/composable_kernel ; https://github.com/ROCm/aiter ; https://github.com/ROCm/jax-aiter ; https://github.com/ROCm/ATOM ; https://github.com/ROCm/aotriton ; https://github.com/ROCm/triton ; https://github.com/ROCm/jax-triton ; https://github.com/ROCm/flash-attention ; https://github.com/ROCm/MAD ; https://github.com/ROCm/pytorch ; https://github.com/ROCm/vllm ; https://github.com/ROCm/triton-inference-server-server ; https://github.com/ROCm/triton-inference-server-core ; https://github.com/AMD-AIG-AIMA/torchtitan-amd ; https://github.com/AMD-AGI/Primus ; https://github.com/AMD-AGI/Nitro-1 ; https://github.com/AMD-AGI/Nitro-T ; https://github.com/AMD-AGI/Nitro-E ; https://github.com/amd/Quark ; https://github.com/amd/quark-documentation ; https://github.com/amd/gaia ; https://github.com/amd/gaia/releases ; https://github.com/amd/gaia/releases/tag/v0.17.0 ; https://github.com/pytorch/torchtitan ; https://github.com/deepspeedai/DeepSpeed ; https://github.com/triton-lang/triton ; https://github.com/openai/triton/tree/rocm ; https://github.com/vllm-project/vllm .
+
+**Container registries**: https://hub.docker.com/r/rocm/pytorch ; https://hub.docker.com/r/rocm/pytorch-training ; https://hub.docker.com/r/rocm/vllm ; https://hub.docker.com/r/rocm/vllm-dev ; https://hub.docker.com/r/rocm/deepspeed ; ROCm Primus image `docker.io/rocm/primus:v26.2` ; HF TGI ROCm `ghcr.io/huggingface/text-generation-inference:latest-rocm` ; ROCm wheels mirror `https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2.1/` ; PyTorch ROCm 7.2 nightly index `https://download.pytorch.org/whl/nightly/rocm7.2` .
+
+**Quark, Quantization, Tooling**: https://quark.docs.amd.com/latest/ ; https://quark.docs.amd.com/latest/intro.html ; https://pypi.org/project/amd-quark/ ; https://www.amd.com/en/developer/resources/technical-articles/amd-quark-quantizer-for-efficient-ai-model-deployment.html ; https://docs.vllm.ai/en/stable/features/quantization/quark/ ; https://docs.vllm.ai/en/stable/getting_started/installation/gpu/ ; https://docs.vllm.ai/en/latest/getting_started/amd-installation.html ; https://pytorch.org/get-started/locally/ ; https://pytorch.org/docs/stable/fsdp.html ; https://www.deepspeed.ai/ .
+
+**Ryzen AI / Edge / GAIA (peripheral but referenced)**: https://www.amd.com/en/developer/resources/ryzen-ai-software.html ; https://ryzenai.docs.amd.com/en/latest/index.html ; https://ryzenai.docs.amd.com/en/1.6.1/model_quantization.html ; https://amd-gaia.ai/docs ; https://www.amd.com/en/developer/resources/technical-articles/gaia-an-open-source-project-from-amd-for-running-local-llms-on-ryzen-ai.html .
+
+**Networking / Pensando**: https://www.amd.com/en/products/network-interface-cards/pensando.html ; https://www.amd.com/en/solutions/data-center/networking.html ; https://www.amd.com/en/blogs/2024/transforming-ai-networks-with-amd-pensando-pollar.html .
+
+**Hugging Face — AMD models, datasets, and the live hackathon org**: https://huggingface.co/amd ; https://huggingface.co/amd/AMD-Llama-135m ; https://huggingface.co/amd/AMD-Llama-135m-code ; https://huggingface.co/amd/AMD-OLMo ; https://huggingface.co/collections/amd/amd-olmo ; https://huggingface.co/amd/Instella-3B ; https://huggingface.co/amd/Instella-3B-Instruct ; https://huggingface.co/amd/Instella-3B-Long-Instruct ; https://huggingface.co/amd/Instella-VL-1B ; https://huggingface.co/amd/Nitro-T-0.6B ; https://huggingface.co/amd/Nitro-E ; https://huggingface.co/datasets/amd/Instella-GSM8K-synthetic ; https://huggingface.co/lablab-ai-amd-developer-hackathon ; https://www.amd.com/en/blogs/2024/introducing-amd-nitro-diffusion--one-step-diffusi.html .
+
+**Prior AMD hackathons (study material)**: https://www.amd.com/en/developer/resources/2024-pervasive-ai-developer-contest-winners.html ; https://www.hackster.io/contests/amd2023 ; https://www.amd.com/en/developer/resources/technical-articles/2025/amd-open-robotics-hackathon-recap.html ; https://www.amd.com/en/developer/resources/technical-articles/2025/hack-the-edge-amd-and-liquid-ai-hackathon-recap.html ; https://www.amd.com/en/developer/resources/technical-articles/2026/amd-ai-reinforcement-learning-hackathon-recap.html ; https://www.amd.com/en/developer/resources/technical-articles/2026/new-gpumode-virtual-hackathon--e2e-model-speedrun.html ; https://lablab.ai/ai-hackathons/anthropic-ai-hackathon ; https://lablab.ai/event/mistral-7b-24-hours-hackathon ; https://lablab.ai/apps/tech/mistral-ai ; https://lablab.ai/ai-hackathons/nano-payments-arc ; https://lablab.ai/ai-hackathons/openclaw-surge-hackathon ; https://lablab.ai/ai-hackathons/ai-trading-agents-erc-8004 ; https://lablab.ai/ai-hackathons/milan-ai-week-hackathon .
+
+**User's existing assets (PYTHAI/DELTAVERSE ecosystem)**: https://mindx.pythai.net (cognitive AI API), https://agenticplace.pythai.net (agent marketplace), https://agenticplace.pythai.net/allchain.html (chainmapping directory), https://bankon.pythai.net (identity / x402 / Algorand / ENS subnames), https://pythai.net , https://rage.pythai.net , https://gpt.pythai.net , https://github.com/pythaiml/automindx , https://github.com/Professor-Codephreak , https://github.com/pythaiml , https://github.com/Professor-Codephreak/automind/ , https://rage.pythai.net/professor-codephreak-2/ , https://rage.pythai.net/easyagi/ , https://rage.pythai.net/autotrain/ .
+
+## Closing read
+
+codephreak — the hackathon's structural shape rewards exactly what mindX already is: a multi-agent cognitive system with a marketplace surface and a payment plane. The missing piece, **xtrain**, is also the piece that makes the fine-tuning track winnable rather than a shallow LoRA wrapper. The integrated submission lets you contest all three primary tracks plus the x402 challenge plus Build-in-Public from one repo, one demo URL, and one MI300X droplet, with the four lablab judging axes already mapped onto your README sub-sections. The two highest-leverage bets in your remaining time are: keep the MI300X droplet up continuously through the 72-hour judging window, and ship Build-in-Public posts on a strict cadence with the four required tags. The asymmetric upside is the **Best Overall** prize, which a full-stack play wins by structural definition over single-track entries; the realistic floor is winning two of three primary tracks plus the x402 side pool plus Build-in-Public. The on-chain artifacts (ERC-7857 INFTs on Base, x402 Algorand metering, Lighthouse-pinned checkpoints) make the submission verifiable from the block explorers, which moves the project from "demo theater" to "production system the judges can audit live" — the single highest-signal credibility move in lablab's recent pattern of winning entries. Build clean, ship early, keep the demo live, and let the integrated architecture do the talking.

docs/blueprints/mindXtrain2.md

@@ -0,0 +1,390 @@
+# mindXtrain — GLM-5.1, aGLM Lineage & Training-Framework Master Reference
+
+> Foundational technical brief prepared for Gregory ("codephreak" / Professor Codephreak), BANKON / PYTHAI / DELTAVERSE, May 2026. Apache 2.0 redistribution target. Python ≥ 3.12. Podman, OpenBSD vmm, Foundry. Flat snake_case, cypherpunk2048 standard.
+
+This document is the working reference for the mindXtrain project — the training framework that will produce aGLM v2 derivatives consumable by automindX v2 and the cognitive API at `mindx.pythai.net`. It corrects three premise errors in the original research brief, executes the full technical analysis the brief requested, and ends with a concrete construction plan and a non-trivial recommendation: **target Qwen3.5 as the primary base and treat GLM-5.1 as a premium specialist track**, on rigorous evidence laid out below. None of this is hand-waved; every figure is sourced and every disagreement between sources is flagged.
+
+## Three corrections to the brief, before anything else
+
+The first correction is that **the GLM-5.1 "family" the brief assumed does not exist**. There is no GLM-5.1-Air, no GLM-5.1-Flash, no GLM-5.1-AirX, no GLM-5.1-Plus, and no GLM-5.1V. The Hugging Face collection at `zai-org` contains exactly two artifacts — `zai-org/GLM-5.1` (BF16 weights) and `zai-org/GLM-5.1-FP8` (native FP8 quantization). The Z.AI documentation sidebar lists a single GLM-5.1 entry. The Ollama library exposes one tag, `glm-5.1:cloud`, which routes to Z.AI's hosted endpoint rather than running locally. Earlier-generation tiers (GLM-4.5-Air, GLM-4.5-Flash, GLM-4.7-Flash, GLM-4.7-FlashX) remain accessible on the Z.AI API but do not share GLM-5.1's `glm_moe_dsa` architecture and are not part of the same training generation. The "Air/Flash" cost-tier idea from GLM-4.5 was deliberately abandoned for GLM-5.1: Z.AI shipped one flagship designed to do everything, plus an FP8 quantization to make it deployable on a single 8×B200 node. Any planning that depends on a smaller native GLM-5.1 variant has to either use community quantizations, fall back to GLM-4.5-Air for the cheap tier, or pick a different model family for the lower rungs of the size ladder.
+
+The second correction is that **the "GLM" inside aGLM is not Zhipu GLM**. Reading `pythaiml/automindx/aglm.py` and the `autoGLM/README-md` concept document together makes this unambiguous: aGLM stands for *Autonomous General Learning Model* (also rendered *Autonomous General Learning Machine* — both are author-sanctioned). The actual model loaded by `aglm.py` in its current canonical form is `TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML`, a Llama-2 GGML-era quant — not a ChatGLM or GLM-4 weight. The acronym collision with Zhipu's separate "AutoGLM" research project (`zai-org/Open-AutoGLM`, the AutoGLM-Phone-9B paper) is a permanent disambiguation hazard the v2 README must address up front. mindXtrain v2's job is therefore *not* to "wrap GLM-5.1 inside aglm.py" in the literal sense; it is to upgrade the aGLM runtime so that one of the model backends it can dispatch to is GLM-5.1 (or Qwen3.5, or any modern base), while preserving the Codephreak persona, agenda-conditioning, four-axis decomposition, and JSON-on-disk memory pattern that constitute aGLM's identity.
+
+The third correction is that **`huggingface/ml-intern` is not a training framework**. The repository, first published around 19–21 April 2026 by Aksel Joonas Reedi and the HF AI-Agents team, is a Claude-Code-style autonomous coding agent pre-wired to Hugging Face Hub, Papers, Datasets, Jobs, and Spaces. Its `pyproject.toml` declares `name = "hf-agent"` at version 0.1.0 and lists `huggingface-hub>=1.0.1`, `litellm>=1.83.0`, `fastmcp>=3.2.0`, `pydantic>=2.12.3`, and `whoosh>=2.7.4` — but it does **not** depend on `transformers`, `peft`, `trl`, `accelerate`, `lighteval`, or `evaluate`. Distributed training is delegated entirely: ml-intern's `agent/tools/hf_jobs.py` writes a TRL-or-Transformers script in-context, submits it to a Hugging Face Job at the chosen flavor (`gpu-h100`, `gpu-a100`, etc.), polls the logs, and feeds them back into the LLM context. There is no Trainer abstraction inside ml-intern. There is also, as of late April 2026, **no LICENSE file** — issue #41 in the ml-intern repo is the open ticket asking HF to confirm; the third-party `mudler/universal-ml-intern` port assumes Apache 2.0, but that is an assumption, not an attached license. mindXtrain cannot vendor ml-intern code yet. What it **can** do — and what this document recommends — is study and reimplement five specific patterns from ml-intern (the unified `ToolRouter`, the bounded ReAct loop with doom-loop detector, the 170k-token auto-compacting `ContextManager` with Claude-Code-JSONL trajectory upload, the approval-required tool flag with live USD pricing, and the three-phase Research → Plan → Implement system prompt). These patterns belong in the *operator layer* above mindXtrain's training core, not inside it.
+
+With those corrections in place, the rest of this document is exhaustive on the technical substance.
+
+## Part 1 — GLM-5.1 in full technical depth
+
+GLM-5.1 was announced via the Z.AI blog post *"GLM-5.1: Towards Long-Horizon Tasks"* (`https://z.ai/blog/glm-5.1`) and released on 7 April 2026. It is not a new pretraining run; it is a post-training refresh of GLM-5, sharing the same `glm_moe_dsa` architecture and the same paper, *"GLM-5: from Vibe Coding to Agentic Engineering"* (arXiv 2602.15763, lead author Aohan Zeng, 185-author roster). The differentiator that justifies the .1 bump, per the Z.AI blog and the model card on `huggingface.co/zai-org/GLM-5.1`, is asynchronous agent-RL training "emphasizing hundreds of rounds and thousands of tool calls" — agentic-trajectory reinforcement learning with verifiable rewards on real engineering tasks (the Z.AI team cites optimizing a vector database to 21,500 QPS over 600+ iterations and 6,000 tool calls, and KernelBench Level 3 producing a 3.6× geometric-mean speedup vs `torch.compile max-autotune`'s 1.49× over thousands of optimization rounds). The marketing tagline — "can work autonomously on a single task for up to 8 hours" — is a deliberate framing of where the model's headroom is.
+
+### Parameter accounting and shape
+
+GLM-5.1 is a Mixture-of-Experts decoder of total capacity ~754 B parameters with ~40.8 B active per token. The 754 B figure is what the Hugging Face model card metadata reports; Lambda's deployment guide and the GLM-5 GitHub README report 744 B; the gap (~10 B) is reconciled by the static reference site `glm51.si5.pl` — built directly from the merged `transformers/models/glm_moe_dsa/{configuration,modular,modeling}_glm_moe_dsa.py` — as MTP head plus fp32 buffers plus per-checkpoint scale tensors. Active-per-token is consistent across all sources at "~40 B".
+
+The model has 78 decoder layers. The first three are dense; the remaining seventy-five are MoE — the configuration object encodes this as `mlp_layer_types = ["dense"] * 3 + ["sparse"] * 75`. Hidden size `d_model` is 6,144. Vocabulary is 154,880 tokens (up from 151,552 in GLM-4.6); embeddings and lm_head are untied (`tie_word_embeddings=False`), so each is a 6,144 × 154,880 matrix of about 951.4 M parameters, contributing roughly 1.9 B parameters to the total just for the input/output projections. Native context length is 202,752 tokens — about 200 K — with no YaRN or NTK extrapolation in the released config. RoPE is the NeoX/Llama split-half variant (rotate-half), applied only to the 64-dimensional "rope" subspace via `attribute_map = {"head_dim": "qk_rope_head_dim"}`; `rope_theta` is configurable in `rope_parameters` and `partial_rotary_factor` is honored. Crucially, GLM-5.1 explicitly **removed** the interleaved-RoPE path GLM-5 inherited from DeepSeek V3 — `rope_interleave` raises `AttributeError`. Any weight-conversion script written for GLM-5 that assumes the DeepSeek V3 RoPE will silently break on GLM-5.1.
+
+Normalization is RMSNorm pre-norm, weight-only, ε = 1e-5 throughout the body and inside the attention's `q_a_layernorm` and `kv_a_layernorm`. The lone exception is the DSA Indexer's `k_norm`, which is a standard `nn.LayerNorm` with ε = 1e-6 — a deliberate departure to match the DeepSeek V3.2 reference implementation. Activations are SiLU inside a SwiGLU GLU MLP (`gate_proj`, `up_proj`, `down_proj`, no biases). Attention bias is False everywhere. The FP8 escapes are explicit: `_keep_in_fp32_modules = ["indexer.weights_proj"]` and `_keep_in_fp32_modules_strict = ["e_score_correction_bias"]`.
+
+### Attention: MLA stacked with DSA
+
+Every layer combines two attention innovations. The first is Multi-head Latent Attention from DeepSeek V2/V3: 64 query heads, 64 KV heads (in MLA, KV are produced from a low-rank latent and expanded per head, so `num_key_value_groups` is 1 and `repeat_kv` is a no-op), a query LoRA rank of 2,048 (raised from 768 in GLM-5 because the q-latent must now also feed the new DSA Indexer), a KV LoRA rank of 512, an `nope` content slice of 192 dimensions per head and a `rope` positional slice of 64 dimensions per head for total `qk_head_dim = 256`, and a value head dimension of 256. The latent KV cache is therefore 512 + 64 = 576 elements per token per layer, or roughly 89.9 KB per token over the 78 layers; the *expanded* KV cache that vanilla Transformers materializes is 64 × (256 + 256) = 32,768 elements per token per layer, about 5.13 MB per token. That ratio is the entire reason MLA exists.
+
+The second innovation is the DeepSeek Sparse Attention Indexer, the headline change from GLM-5. The Indexer has 32 heads of 128 dimensions each, takes the post-`q_a_layernorm` query latent for queries, reads raw `hidden_states` through its own `wk` projection for keys, computes scores as `Σ_h weights[s,h] · ReLU(softmax_scale · q[s,h] · k[t])` in fp32, and selects `index_topk = 2,048` keys per query. The Indexer adds about 9.4 M parameters per layer (~5.4% of the layer's attention weight) and maintains its own KV cache outside the standard `DynamicCache` as a layer-local `_cached_keys` tensor (~256 B per token in bf16). The flash-mla kernel from `kernels-community/flash-mla` reads the `topk_indices` kwarg directly to skip masked positions; the eager and SDPA paths instead materialize a full `[B, S, T]` `-inf` mask and `scatter_` zeros at the indexer-selected columns. The point of the Indexer is that it makes per-token attention compute **independent of sequence length** beyond about 2 K, which is what makes 200 K-token decode tractable at 754 B-parameter scale. Note that DSA does *not* shrink the KV cache — it shrinks compute. KV cache memory at long context is still dominated by MLA's latent representation.
+
+### MoE block
+
+The 75 sparse layers carry 256 routed experts plus 1 always-on shared expert with intermediate size 2,048; top-k routing selects 8 routed experts per token, so 8 + 1 = 9 experts are active at any moment. The dense FFN in layers 0–2 has intermediate size 12,288 (about 2× hidden). Router scoring uses sigmoid (not softmax) in fp32, with auxiliary-loss-free balancing via a per-expert `e_score_correction_bias` that is added for *selection only*, not for weighting — the DeepSeek V3 trick. `routed_scaling_factor` is 2.5 (raised from 1.8 in GLM-5), multiplied into the L1-normalized top-k sigmoid scores before they are added to the residual. The shared-expert path is added without the routing scale, so it acts as a constant baseline. The `n_group`/`topk_group` fields are 1/1, which collapses the multi-group routing back to plain top-8 over all 256 experts. Per-MoE-layer capacity is 256 × 37.75 M (3 × 6144 × 2048) + 1 × 37.75 M + 1.57 M router ≈ 9.70 B; per-MoE-layer active is 8 × 37.75 M + 37.75 M ≈ 341.3 M FFN + 174.4 M attention ≈ 515.7 M per token per layer. Aggregate: ~743.6 B capacity, ~40.8 B active, matching the official "754 B / ~40 B" within the noted reconciliation gap.
+
+### MTP head, tokenizer, chat template
+
+GLM-5.1 ships a Multi-Token Prediction head for speculative decoding, DeepSeek V3 style. The Hugging Face transformers integration uses `_keys_to_ignore_on_load_unexpected = [r"model\.layers\.78.*"]` to admit a layer-78 placeholder for the MTP head; the main body has 78 layers indexed 0–77. vLLM exposes MTP via `--speculative-config.method mtp --speculative-config.num_speculative_tokens 3`; SGLang uses EAGLE for the same role. The tokenizer is a SentencePiece-derived BPE with 154,880 vocabulary entries; special tokens include `<|system|>`, `<|user|>`, `<|assistant|>`, `<|observation|>`, `<|tool|>`, plus thinking delimiters and tool-call tokens. vLLM's `--tool-call-parser glm47 --reasoning-parser glm45` flags select the right parser pair. **Thinking mode is enabled by default** in GLM-5.1 — a behavioral change from GLM-5 — and is disabled with `chat_template_kwargs: {"enable_thinking": false}`. The `chat_template.jinja` (~4.67 kB) supports Claude-style deferred tool loading via `defer_loading=True`, which puts tool schemas into the tool *result* messages rather than the system prompt, and accepts both `List[tool]` and `List[tool.function]` shapes for SGLang compatibility. There are two thinking flavors: *Interleaved Thinking* (default, general chat) and *Interleaved + Preserved Thinking* for agentic workflows like Claude Code, Roo Code, and Kilo Code, toggled via the `enable_thinking` and `clear_thinking` chat-template kwargs.
+
+### Training methodology, with explicit gaps
+
+Pretraining used 28.5 trillion tokens, up from 23 T for GLM-4.5. The English:Chinese:other split is not disclosed in the model card or the extractable arXiv abstract; the tokenizer is jointly built over English and Chinese with code and tool tokens, and the Hugging Face metadata tags the model with both `English` and `Chinese`. One secondary blog (`aimadetools.com`) claims pretraining was done on 100,000 Huawei Ascend 910B chips with zero NVIDIA dependency; this is **not** corroborated by the arXiv paper, the model card, or the Z.AI blog and should be treated as unverified reporting, although the parallel choice of `xLLM` (JD's Ascend-aware serving stack) as a first-class deployment target lends it some weight. Total pretraining FLOPs are not disclosed.
+
+Post-training is described in the arXiv 2602.15763 abstract — the only programmatically extractable text from the paper at the time of this research; the PDF body is not machine-readable in current archives — as using an asynchronous reinforcement-learning infrastructure called `slime`, open-sourced at `github.com/THUDM/slime`. `slime` decouples generation from training so that fine-grained RL iterations can run without blocking the trainer. The asynchronous agent-RL algorithms are said to "improve RL quality, enabling the model to learn from complex, long-horizon interactions more effectively." For GLM-5.1 specifically, the differentiator is that this RL was run with verifiable rewards over real engineering tasks at "hundreds of rounds and thousands of tool calls" depth. The choice of PPO vs GRPO vs DPO, the SFT data mix, the reward-model architecture, the cold-start reasoning data, and the source of any distillation are **not disclosed**. mindXtrain's RL track will therefore have to either trust `slime` as the operational primitive (it is GPL-3.0 / Apache 2.0 / MIT compatible — verify per repo file) or stand up its own GRPO/DPO loops via TRL.
+
+### Benchmarks: vendor-claimed vs independent
+
+Z.AI's published numbers for GLM-5.1 are deliberately concentrated on long-horizon agentic and engineering benchmarks — SWE-Bench Pro 58.4 (claimed SOTA, 0.7 points above GPT-5.4's 57.7 and 1.1 above Claude Opus 4.6's 57.3), Terminal-Bench 2.0 63.5 with Terminus-2 / 66.5 with Claude Code (vs Gemini 3.1 Pro at 68.5), CyberGym 68.7 (claimed SOTA over Claude Opus 4.6 at 66.6), BrowseComp 68.0 (claimed SOTA), MCP-Atlas 71.8, τ³-Bench 70.6, Tool-Decathlon 40.7, Vending Bench 2 final balance \$5,634, AIME 2026 95.3, HMMT February 2026 82.6, GPQA-Diamond 86.2, HLE 31.0 / HLE w/ Tools 52.3. Z.AI explicitly did not publish numbers for MMLU, MMLU-Pro, CMMLU, C-Eval, BBH, MATH-500, GSM8K, HumanEval, MBPP, LiveCodeBench, BigCodeBench, BFCL v3, AgentBench, GAIA, vanilla SWE-bench, IFEval, Arena-Hard, RULER, or NIAH. If mindXtrain needs any of those, they have to be re-run locally.
+
+The independent picture is more mixed. Artificial Analysis's Intelligence Index v4.0 places GLM-5.1 (Reasoning) at 51 and GLM-5.1 (Non-Reasoning) at 44 — well above the open-weight reasoning median of 29 but a clear step below GPT-5.4 / Gemini 3.1 Pro / Claude Opus 4.6, which sit closer to 65–75. AAII v4.0 evaluation consumed about 110 M output tokens (vs a 40 M peer median, "very verbose") at \$543.95 total. BenchLM's *provisional* leaderboard ranks GLM-5.1 #14 of 115 models with an overall 83 — but its *verified* leaderboard, which only counts source-attached scores, places GLM-5.1 #21 of 23. That gap is the cleanest signal that vendor-reported scores are running ahead of independently re-run scores. LiveBench had not yet ingested GLM-5.1 in the captures available at the time of research; LMSYS Chatbot Arena had not surfaced a GLM-5.1 ELO yet (GLM-5 base sits around 1451 and was reported as the #1 open model in the GLM-5 paper); HuggingFace Open LLM Leaderboard v2 cannot evaluate models of this size under its compute budget. The Scale AI SEAL leaderboard hosts the SWE-Bench Pro 58.4 number with the asterisk denoting self-submission and no independent re-run as of this report. The fair characterization is that GLM-5.1 is genuinely a top-tier open-weights model that sits within roughly 5–10% of the absolute frontier closed models on most coding benchmarks, narrowly leads on SWE-Bench Pro (self-reported), and is the strongest open agentic-coding model available in May 2026 — but the Z.AI marketing claim of "outperforming GPT-5.4, Claude Opus 4.6, and Gemini 3.1 Pro" is true on a single benchmark and false on most others. Cherry-picked headline.
+
+### Inference characteristics and VRAM math worked out
+
+Weight memory at 754 B parameters: BF16 ≈ 1,508 GB (what `zai-org/GLM-5.1` ships, marked "BF16 · F32" in the HF metadata); native FP8 ≈ 754 GB (what `zai-org/GLM-5.1-FP8` ships — the on-disk checkpoint is 756 GB across 142 safetensors shards of about 5.36 GB each); INT4 ≈ 377 GB (community quants, currently one such model listed under HF "Quantizations"); INT8 ≈ 754 GB (rare — not commonly distributed). GGUF Q4_K_M / Q5_K_M / Q6_K / Q8_0 are **not available** because llama.cpp does not yet ship the `glm_moe_dsa` architecture; Unsloth has shipped Dynamic 2.0 GGUF quants in the UD-IQ2_M (~241 GB) through UD-Q8_0 range at `huggingface.co/unsloth/GLM-5.1-GGUF` by carrying a ggml-org/llama.cpp build, but the upstream PR numbers landing the architecture were not surfaced in the research pass and the architecture is closely related to GLM-4.5/4.6.
+
+KV cache at batch 1 in BF16, expanded form: about 3.7 GB at 4 K tokens, 30 GB at 32 K, 120 GB at 128 K, 187 GB at 200 K. In MLA-compressed (latent) form: 0.36 GB at 4 K, 2.9 GB at 32 K, 11.5 GB at 128 K, 18 GB at 200 K. Total VRAM for FP8 weights plus KV at 128 K context: 754 + 120 = 874 GB (expanded) or 754 + 11.5 = 765 GB (MLA-compressed). For INT4 + MLA-compressed KV: 377 + 5.7 = 383 GB. With DSA active on a ~2 K window, FP8 + 0.18 GB = 754 GB. The official Lambda minimum is **a single 8×B200 (HGX B200) node** to load the FP8 model with usable context. **An RTX 4090 24 GB cannot run GLM-5.1 at any quantization.** **An M3 Ultra 192 GB unified-memory Mac cannot run the full model at any quantization** — the FP8 weights alone are 754 GB; in principle the INT4 quant would fit if MLX support existed, but `ml-explore/mlx-lm` issue #879 ("Add model support for GLM-5 (`glm_moe_dsa` architecture)") was still open at research time. **An M2 Max 96 GB cannot run any usable form of GLM-5.1.** A100 80 GB requires roughly 10× to load FP8.
+
+Throughput, from Lambda's official benchmarks at 8 K input / 2 K output and 32 concurrent users: SGLang 0.5.10 on 1× HGX B200 produces 1,345.4 output tokens/s, 42.0 per-user, 6,727.2 total, with mean TTFT 1,073 ms and mean inter-token latency 58.6 ms. vLLM 0.19.0 on the same hardware produces 1,265.4 / 39.5 / 6,327.2 tokens/s with TTFT 1,317 ms, ITL 57.8 ms. Artificial Analysis's median across providers is 53.7 tok/s output with TTFT 1.73 s — slower than the open-weight median of 56.3 tok/s in this scale class.
+
+The recommended decoding parameters from the Z.AI model card are: temperature 1.0, top_p 0.95, max generation 131,072 for default and general use; temperature 0.7, top_p 1.0, max 16,384, with `enable_thinking=true` and `clear_thinking=false` for Terminal-Bench / coding-agent flows; temperature 0, max 16,384, Interleaved + Preserved thinking for τ²-Bench-style pure tool-use flows.
+
+### Framework support matrix
+
+Hugging Face Transformers requires version 5.3.0 or later — the `glm_moe_dsa` architecture is not in transformers 4.x. (Z.AI's README states "v0.5.3+", which is a typographical error for v5.3.0+; Lambda's deployment guide uses the corrected version.) `trust_remote_code` is no longer required after the merge. vLLM supports GLM-5.1 from 0.19.0 with custom Docker images at `vllm/vllm-openai:glm51` and `vllm/vllm-openai:glm51-cu130` (CUDA 13+); the recipes URL is `https://docs.vllm.ai/projects/recipes/en/latest/GLM/GLM5.html`. There is a known caveat: tool-calling combined with MTP-enabled speculative decoding requires the vLLM main branch rather than the 0.19.0 release. SGLang requires v0.5.10 (not 0.5.10rc0, which has a known flashmla bug fixed in the 0.5.10 release). xLLM v0.8.0+ supports the Ascend NPU path. KTransformers v0.5.3+ supports CPU-offload-plus-GPU hybrid. Ollama supports `glm-5.1:cloud` (cloud-only) as of research time; an open issue at `github.com/ollama/ollama/issues/15412` tracks offline support. A community fork at `ollama.com/frob/glm-5.1` imports `unsloth/GLM-5.1-GGUF` and requires a patched Ollama build with PR #14864 applied; the maintainer notes tool-calling is poor on this fork pending an Ollama PARSER. **Important known bug:** Unsloth's discussion thread on `huggingface.co/unsloth/GLM-5.1-GGUF/discussions/4` warns that CUDA 13.2 produces gibberish or breaks tool calling on Gemma 4 and GLM-5.1; NVIDIA had not issued a fix at research time. Use CUDA 13.0 or 13.1 for any GGUF deployment.
+
+A canonical vLLM launch and a canonical SGLang launch, copy-pasteable:
+
+```bash
+vllm serve zai-org/GLM-5.1-FP8 \
+  --tensor-parallel-size 8 \
+  --max-model-len 202752 \
+  --max-num-seqs 64 \
+  --speculative-config.method mtp \
+  --speculative-config.num_speculative_tokens 3 \
+  --tool-call-parser glm47 \
+  --reasoning-parser glm45 \
+  --enable-auto-tool-choice \
+  --chat-template-content-format=string \
+  --served-model-name glm-5.1-fp8
+```
+
+```bash
+SGLANG_ENABLE_SPEC_V2=1 \
+sglang serve \
+  --model-path zai-org/GLM-5.1-FP8 \
+  --tp-size 8 \
+  --tool-call-parser glm47 \
+  --reasoning-parser glm45 \
+  --speculative-algorithm EAGLE \
+  --speculative-num-steps 3 \
+  --speculative-eagle-topk 1 \
+  --speculative-num-draft-tokens 4 \
+  --mem-fraction-static 0.85 \
+  --served-model-name glm-5.1-fp8
+```
+
+### API access
+
+The English portal is `https://api.z.ai/api/paas/v4/`, with OpenAI-compatible chat completions at `POST /chat/completions`, `Authorization: Bearer <key>`, keys managed at `https://z.ai/manage-apikey/apikey-list`. The Chinese portal is `bigmodel.cn`. The model ID is `glm-5.1`; siblings include `glm-5`, `glm-5-turbo`, `glm-4.7`, `glm-4.7-flash`, `glm-4.7-flashx`, `glm-4.6`, `glm-4.5`, `glm-4.5-air`, `glm-4.5-airx`, `glm-4.5-x`, and `glm-4.5-flash`. The Python SDK is `pip install zai-sdk` (≥0.2.2); Java is Maven `ai.z.openapi:zai-sdk:0.3.3`; any OpenAI-compatible client with `base_url="https://api.z.ai/api/paas/v4/"` works. Pricing for GLM-5.1 in USD per 1 M tokens is \$1.40 in / \$0.26 cached input / \$4.40 out, with cache storage free for a limited time. Third-party reseller pricing varies: OpenRouter `z-ai/glm-5.1` is \$1.05 / \$3.50 (output capped at 65,535 tokens); Requesty / Fireworks match the official \$1.40 / \$4.40 (max output 25,000, 202 K context with prompt caching "up to 90%"); Inworld / DeepInfra match OpenRouter at \$1.05 / \$3.50; the Artificial Analysis median is \$1.40 / \$4.40. Feature flags supported on the native API include OpenAI-compatible `tools` arrays (with Claude-style deferred tool loading), MCP integration, structured JSON output, streaming including tool-streaming output, prefix/context caching, the `thinking: {"type": "enabled" | "disabled"}` toggle, and a built-in web search tool at \$0.01 per use. Vision input is **not** supported on `glm-5.1` — use the sibling `glm-5v-turbo`. There is no free tier on `glm-5.1` itself; the free tier exists on the Flash variants of earlier generations.
+
+### License — the most important section for BANKON
+
+GLM-5.1 weights ship under the **MIT License** on Hugging Face. The model card YAML for both `zai-org/GLM-5.1` and `zai-org/GLM-5.1-FP8` declares `license: mit`; the Unsloth GGUF mirror at `huggingface.co/unsloth/GLM-5.1-GGUF` does the same; Wikipedia summarizes Z.ai's policy as "released under the free and open-source MIT License since July 2025." The Hugging Face license-label "mit" maps in HF's taxonomy to the canonical SPDX MIT text — HF rejects uploads that declare `license: mit` if the LICENSE file deviates. Treat this as high-confidence even though the literal raw bytes of `github.com/zai-org/GLM-5/blob/main/LICENSE` returned 429s during the research pass. (Note: there is a code-vs-weights split — the GitHub repo header for `zai-org/GLM-5` reads "Apache License 2.0" in the search index sidebar, which means the *code* in the repo is Apache 2.0 while the *weights* on Hugging Face are MIT. This is the same pattern Z.ai used for GLM-4.5 and GLM-4.6.)
+
+Verbatim, the operative MIT text for the weights is:
+
+```
+MIT License
+
+Copyright (c) 2026 Z.ai (or "ZHIPU AI" as published)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHER WISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```
+
+Clause-by-clause: **commercial use is unrestricted**; the grant explicitly covers "use, copy, modify, merge, publish, distribute, sublicense, and/or sell." **Redistribution is unrestricted** subject to preserving the copyright notice and the permission text. **Derivative works can be released under any license BANKON chooses**, including Apache 2.0 — MIT is permissive, non-copyleft, and does not propagate. **Attribution is the MIT notice preservation, nothing more** — no name-prefix requirement, no "Powered by GLM" badge, no UI credit. **There are no field-of-use restrictions on the weights** — no military prohibition, no surveillance carve-out, no critical-infrastructure exclusion, no biosecurity rider. There is no patent grant (this is MIT's standard weakness vs Apache 2.0; patent rights are at most implied), no trademark grant (so do not use "GLM", "Zhipu", or "Z.ai" marks in BANKON product names in ways that suggest endorsement — naming the derivative `aGLM-BANKON` is borderline, and a more defensible product mark in user-facing UI is something like "MindX" with a model-card technical name of `aGLM-BANKON`), no AUP, no MAU thresholds, and no geographic restrictions encoded in the license itself. Z.ai is on the U.S. Commerce Department Entity List as of January 2025 — that is a sanctions/export-controls matter affecting U.S. companies *transacting with* Z.ai as a corporate counterparty, not a license restriction encoded in the MIT text; the consensus practice is that downloading MIT-licensed open weights is not Entity-List-restricted activity, but BANKON should confirm with counsel.
+
+The hosted-API regime is separate. `https://docs.z.ai/legal-agreement/terms-of-use` governs `api.z.ai`, `BigModel.cn`, and `chat.z.ai` and contains AUP-style language including a prohibition on using outputs "for the development, training, labeling, fine-tuning, optimization, iteration, or similar activities related to external models" or to "develop, train, or enhance algorithms or models that compete with us"; restrictions on services "requiring subject qualification, including but not limited to medical services, legal services" plus "any decision-making behavior, operation of critical infrastructure, transportation technologies, heavy machinery"; and a clause stating "If you suffer damage after training, fine-tuning and development and claim that we should bear the responsibility, you shall prove that the damage is unrelated to your training, fine-tuning and development, otherwise we shall be exempted from liability for the damage." These restrictions are **API-only** and do not bind BANKON if it self-hosts the open weights.
+
+The compliance recipe for BANKON is therefore: pull the FP8 weights from `huggingface.co/zai-org/GLM-5.1-FP8`, self-host on owned/leased GPUs (vLLM or SGLang on a single 8×B200 or two 8×H100s), and avoid touching `api.z.ai` for any production path that would create a contractual nexus. For the `aGLM-BANKON` derivative published under Apache 2.0, ship a `LICENSE` file containing the Apache 2.0 text, ship a `NOTICE` file containing the verbatim upstream MIT notice ("Copyright (c) 2026 Z.ai. Licensed under the MIT License. See LICENSE-MIT-upstream for full text."), ship `LICENSE-MIT-upstream` with the MIT text and Z.ai's copyright assertion, and stamp the model card with "Copyright (c) 2026 BANKON — all rights reserved. Fine-tuned from `zai-org/GLM-5.1`, originally licensed under the MIT License (Copyright (c) 2026 Z.ai)." Dual-licensing is permitted; there is no copyleft pull-through. The derivative does not have to inherit the MIT licence — only the upstream notice must be preserved.
+
+Compared to peers: Qwen3 ships under Apache 2.0 across the entire family — equivalent for redistribution, slightly stronger on patent posture, slightly heavier on NOTICE compliance overhead. DeepSeek V3.2 / R1 are MIT for both code and weights — equivalent. Phi-4 / Phi-4-mini are MIT — equivalent. Llama 3.3 / Llama 4 ship under the Llama Community License with the 700 M MAU clause, the "Built with Llama" name-prefix and badge requirement, and the AUP at `llama.meta.com/llama3_3/use-policy` incorporated by reference — *incompatible* with clean Apache 2.0 downstream redistribution. Gemma 1–3 carried the Gemma Terms of Use with a Prohibited Use Policy that propagates to downstream redistributors — restrictive — but **Gemma 4** (April 2026) flipped to Apache 2.0, becoming compatible. Mistral's flagship line (Mistral Large, Codestral, Mistral Small) used the Mistral Research License (non-commercial) until **Mistral Large 3** (December 2025), which moved to Apache 2.0 along with Ministral 3 and Mistral Small 4. Cohere Command A is CC-BY-NC 4.0 with an Acceptable Use Addendum — non-commercial only — *incompatible*. Falcon 3 ships under TII Falcon License 2.0, Apache-2.0-derived but with an enforceable AUP — most legal teams will treat this as restrictive.
+
+## Part 2 — autoGLM and aGLM lineage as foundational context
+
+The pythaiml/automindx repository at `https://github.com/pythaiml/automindx` (id 686099738, network root for the GATERAGE fork) is 84.0% Python, 13.3% Shell, 2.7% Dockerfile, with 23 commits on `main`, 2 stars, 4 forks, and a flat layout: `aglm.py`, `automind.py`, `memory.py`, `uiux.py`, paired with `algm.md`, `automind.md`, `memory.md`, `uiux.md`, plus `4096chunk.md`, `INSTALL.md`, `DOCUMENTATION.md`, `Dockerfile`, `LICENSE`, `README.md`, `automindx.install`, `chunk4096.py`, `hfUIUX.py`, `hfapp.py`, `hfmemory.py`, and `requirements.txt`. The README declares the project's core composition as `codephreak = uiux.py + memory.py + automind.py + aglm.py` and states the persona explicitly: *"Professor Codephreak is an expert in machine learning, computer science and computer programming."* The runtime entry point is `python3 uiux.py --model_name="TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML" --tokenizer_name="TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML" --model_type="ggml" --save_history --file_name="llama-2-7b-chat-codeCherryPop.ggmlv3.q4_1.bin"`. The license is GPL-3.0 by inheritance (the GATERAGE/aglm fork is explicitly GPL-3.0; the wider codephreak ecosystem — easyAGI, RAGE, MASTERMIND — is uniformly GPL-3.0; the README footer asserts "MASTERMIND (c) codephreak GPLv3 2024"); the LICENSE bytes were not retrievable in the research pass but the inheritance is unambiguous.
+
+`aglm.py` itself is a Hugging Face Transformers wrapper, not an agent. The imports — reconstructed from the Hugging Face `aGLM` org card, which is an authoritative author paraphrase — are `os`, `glob`, `ujson`, `psutil`, `transformers.AutoModelForCausalLM`, `transformers.AutoTokenizer`, and `automind.format_to_llama_chat_style`. The single class is `LlamaModel(model_name, models_folder)`, with methods `initialize_model()` (loads tokenizer and model from `models_folder + model_name` via `AutoTokenizer.from_pretrained` and `AutoModelForCausalLM.from_pretrained`) and `generate_contextual_output(conversation_context)` (which formats via `format_to_llama_chat_style`, tokenizes, runs `model.generate(...)`, and decodes). Module-level helpers are `determine_batch_size()` (uses `psutil.virtual_memory()` against a hard-coded `MAX_MEMORY_USAGE` to decide how many memory-file JSONs to load per batch) and `main()` (globs `memory/*.json`, batches via `determine_batch_size()`, reads with `ujson`, builds `conversation_context`, calls `LlamaModel.generate_contextual_output(context)`, prints). There is no async, no asyncio, no LangChain, no SuperAGI, no AutoGen, no `openai`, no `anthropic`, no `chromadb`, no `faiss`, no `pgvector`. Memory is `glob`-walked from `*.json` files via `ujson`; persistence happens via `memory.save_conversation_memory(...)` writing timestamped JSON files.
+
+The patterns that mindXtrain v2 must preserve are precise. First, the **four-axis decomposition**: `uiux.py` (interface) plus `memory.py` (persistence) plus `automind.py` (prompt/format) plus `aglm.py` (model). Second, the **`.py` paired with `.md` discipline** — every Python module has a sibling Markdown documentation file colocated. Third, **shallow flat class hierarchy**: `LlamaModel`, `DialogEntry`, `EasyAGI`, `AGI`, `LogicTables`, `SocraticReasoning`, `SelfHealingSystem`, `BDI`, `Memory` — no mixins, no ABCs, no Protocols; one responsibility each, instantiated once. Fourth, **synchronous default surface** — `LlamaModel.generate_contextual_output` is sync, `EasyAGI.main_loop` is sync, the whole stack is a blocking REPL or batch. Fifth, **named-class registry dispatched by string key** for tool-like backends (`GPT4o`, `GroqModel`, `OllamaModel` selected by `APIManager` in easyAGI). Sixth, **append-only JSON-on-disk memory with batch-glob replay** ordered by filename timestamp. Seventh, **`psutil`-driven memory budgeting** generalizable to a `ResourceBudget` helper used identically by training, eval, and inference. Eighth, **CLI flags with no config file** — argparse in `uiux.py`, with `automindx.install` shell-script baking the canonical invocation. Ninth, **persona-as-agenda**: the system prompt isn't a role description but contains an explicit *agenda* (the model is told "your job is to build the automindx deployment environment"). The agenda string must remain a first-class field in v2, not baked into a string.
+
+The gaps mindXtrain must fill are equally precise. The hard-coded Llama-2 chat assumption via `format_to_llama_chat_style` must be replaced by a `ChatTemplate` abstraction that picks the right template for Llama-3, Mistral, Qwen, GLM-4, GLM-5.1, and any future base, falling back to `tokenizer.apply_chat_template`. There is no multi-model registry — `LlamaModel` is one class for one family, with a separate dual-path GGML loader implicit in `uiux.py`'s `--model_type="ggml"` branch; v2 needs `class ModelRegistry` with `register(name, factory)` and `get(name)`, supporting backends `hf-transformers`, `llama-cpp-python`, `ollama`, `vllm`, `openai`, `anthropic`, `groq`, and the Z.ai API. The `autoGLM/litellm` fork already foreshadows this; wire it. There is no fine-tuning support — `aglm.py` does inference only — and the `autoGLM/levanter` fork (Apache 2.0, JAX + named tensors) was added as the latent training rail but never integrated. There is no evaluation harness, no observability, no memory layer beyond JSON-on-disk, no tokenizer-aware truncation (the `chunk4096.py` 4096-char ceiling is a workaround for context-window saturation), no session/concurrency primitives, no agent loop in `aglm.py`, no prompt-as-data declarative persona files, no CI, and no tests. Each gap is a real technical debt item, not a feature wishlist.
+
+The wider org context matters because it dictates v2's integration surface. Under `autoGLM/`, the active source repos are `autoGLM/easyAGI` (Python, GPL-3.0, the openmindx → easyAGI point-of-departure stack with modules `EasyAGI`, `AGI`, `LogicTables`, `Reasoning`, `SocraticReasoning`, `SelfHealingSystem`, `Memory`, `GPT4o`, `GroqModel`, `OllamaModel`, `APIManager`, `BDI`); `autoGLM/funAGI` (the first "working" instance with `EasyAGI.main_loop`, archived as the canonical reference); `autoGLM/automindx` (org-level mirror of the canonical `pythaiml/automindx`); and `autoGLM/README-md`, the canonical aGLM concept document defining the architecture as supervised+unsupervised learning with subsystems RAGE (retrieval-augmented memory), machine dreaming, MASTERMIND (logic+prediction), blockchain-anchored knowledge "THOTs" (Theories of Hypothetical Output Trajectories) on decentralized storage, and Continuous Adaptation and Optimization (auto-tuning / self-healing). Forked-in tooling includes `autoGLM/RAGE` (GPL-3.0, retrieval engine), `autoGLM/imaginarium` (TypeScript, NLP UI), `autoGLM/pgvectorscale` (Rust, PostgreSQL-license, the intended long-term-memory backend), `autoGLM/litellm` (the OpenAI-format multi-LLM router), `autoGLM/levanter` (Apache-2.0, JAX-based scalable training — *the latent fine-tuning rail that was never wired into aglm.py*), and `autoGLM/anything-llm` (the desktop/Docker RAG+agent UI candidate). The most complete public surface of the aGLM concept lives at `GATERAGE/aglm`, which is a fork of `pythaiml/automindx` plus MASTERMIND modules (`prediction.py`, `nonmonotonic.py`, `socratic.py`, `reasoning.py`, `logic.py`, `epistemic.py`, `autonomize.py`, `bdi.py`, `terminai.py`, `terminai_module.py`, `SimpleCoder.py`, `model_handler.py`, `controller.py`, `config.json`, `config.py`, `main.py`, plus seventeen numbered `UIUX*.py` evolution snapshots) and is flagged in its own README as "currently BROKEN and useful as reference point of aGLM MASTERMIND and RAGE for modular component display only."
+
+The integration topology is therefore: mindX (`github.com/abaracadabra/mindX`, augmentic-intelligence orchestration, public face `mindx.pythai.net`) is the consumer of the aGLM v2 inference API; PYTHAI hosts the canonical `automindx`, `funAGI`, `pgvectorscale` and the domain anchors `ai.pythai.net`, `gpt.pythai.net`, `rage.pythai.net`, `bankon.pythai.net`, `agenticplace.pythai.net`; MASTERMIND is the rational-controller layer above aGLM (intended composition `MASTERMIND(aGLM, RAGE, BDI)`); RAGE / GATERAGE provides retrieval-augmented memory; DELTAVERSE supplies decentralized/metaDAO settlement; BANKON supplies banking-OS infrastructure (`bankonOS`, `BANKONPYTHAI` with the Algorand ASA `203977300`, the ERC-8004 Identity Registry at `0x8004A169FB4a3325136EB29fA0ceB6D2e539a432` and Reputation Registry at `0x8004BAa17C55a88189AE136b182e5fdA19dE9b63`); Lighthouse Storage / Filecoin is the decentralized-knowledge-storage target named in `autoGLM/README-md`; and `cypherpunk2048/x402` provides the HTTP-402 micropayment protocol for paid inference settlement. mindXtrain produces `aGLM-BANKON` checkpoints, anchors their provenance via ERC-8004 attestations and Lighthouse-Filecoin CIDs, and ships them into automindX v2 which is consumed by mindX at `mindx.pythai.net`.
+
+## Part 3 — `huggingface/ml-intern` patterns to adopt for the operator layer
+
+The five highest-leverage patterns from ml-intern, all of them in the *operator* layer above the trainer, are these.
+
+The **`ToolRouter` and `ToolSpec` dispatcher** (`agent/core/tools.py`, ≈230 LOC) unifies built-in async handlers, MCP JSON-RPC, and OpenAPI specs behind one OpenAI-compatible tool schema, with a deny-list (`{"hf_jobs", "hf_doc_search", "hf_doc_fetch", "hf_whoami"}`) that prevents MCP-supplied tools from shadowing optimized built-ins, and graceful MCP-error capture that returns errors as strings rather than bubbling exceptions. The dataclass is `@dataclass class ToolSpec: name: str; description: str; parameters: dict; handler: Callable | None; needs_approval: bool = False`. mindXtrain's heterogeneous backends — start-run, eval-checkpoint, push-to-hub, deploy-to-API, anchor-to-IPFS, mint-ERC-8004-attestation — map cleanly onto this single surface.
+
+The **bounded ReAct loop with explicit doom-loop detector** (`agent/agent_loop.py`, `submission_loop` and `Handlers.run_agent`) caps autonomous iterations at 300 and runs `DoomLoopDetector.observe(resp.tool_calls)` after each LLM turn; when repeated tool-call patterns (same tool, same args N times) are detected, a corrective system message is injected into the next call to break the cycle. Without this, autonomous training agents wedge inside the first hour of a long run.
+
+The **`ContextManager` with 170 K auto-compaction and Claude-Code-JSONL trajectory upload** (`agent/core/context.py`) handles two responsibilities: it triggers `MessageCompactor` summarization when the message buffer crosses ~170 K tokens, and it serializes every session in Claude-Code JSONL format and pushes it to a private Hugging Face Dataset (`{username}/ml-intern-sessions`). HF's Agent Trace Viewer auto-renders these. For mindXtrain, the JSONL format is the right schema for an auditable, replayable, RFT-corpus-ready training-record format; just swap the storage backend from HF Dataset to a `StorageProvider` interface that writes equally to local FS, HF Datasets, IPFS, or Lighthouse-Filecoin.
+
+The **approval-required tool flag with live USD pricing** combines a per-`ToolSpec` `needs_approval: bool` flag with a generic `await session.await_approval(tc)` flow wired to interactive CLI prompt, web "approve in browser" button, and Slack interactive approval. Live USD/hour pricing is surfaced at the prompt before the user approves any paid operation. For mindXtrain, this is exactly the right ergonomics for "agent picks GPU flavor → user sees \$/hour → user confirms → job runs → logs streamed back into context."
+
+The **three-phase Research → Plan & Validate → Implement system prompt** (`agent/prompts/system_prompt_v3.yaml`) is the load-bearing discipline document. It forces a research phase (read papers, fetch documentation, enumerate options), a plan-and-validate phase (does this dataset exist, does this tokenizer match, does this flavor have the VRAM), and only then an implement phase. The single biggest reason ml-intern's GPQA demo completes in under ten hours is that the LLM is *not allowed* to call paid tools without first emitting and getting approval on a plan. mindXtrain's training agent must inherit this prompt verbatim, with the validation checklist extended to include base-model-hash-pinning, dataset-CID-pinning, and tokenizer-vocab-hash matching against the dataset's tokenized cache.
+
+What ml-intern does **not** provide and mindXtrain must add: decentralized storage (no Filecoin/Lighthouse abstraction — everything goes to HF Hub); blockchain provenance (no on-chain attestation, no signed run-manifests, no Merkle-rooted training-data commitments); first-class hyperparameter sweeps (no Optuna, no Ray Tune — ablations are LLM-driven re-launches); BFCL and agentic-trajectory evals (the `eval` extra ships `inspect-ai>=0.3.149` but BFCL is not first-class); auto-populated model cards from training runs (the agent can be prompted to write a card but there is no card-templating that reads `TrainingArguments` + eval JSON); a hot-swap deployment plane to a live API (push-to-hub exists, but atomic API-level model swap with canary and rollback does not); a checkpoint registry with diffing and promotion; a typed `TrainingRun` Pydantic record as the canonical unit of provenance. All of these go into the mindXtrain trainer and observability layers, not the agent layer.
+
+## Part 4 — mindXtrain technical specification
+
+mindXtrain is the production-grade training framework that produces `aGLM-BANKON-*` derivatives. It is built on the cypherpunk2048 standard (no proprietary lock-in, Podman over Docker, OpenBSD vmm over VirtualBox, Foundry as canonical Solidity test framework, flat snake_case layout, Python ≥ 3.12, Apache 2.0 license with `(c) 2026 BANKON — all rights reserved` plus upstream MIT NOTICE preservation). The framework has three layers: the **trainer core** (Trainer + TRL + PEFT + Accelerate + Transformers, hand-built), the **operator layer** (ml-intern-pattern-derived agent runtime, reimplemented), and the **provenance layer** (Lighthouse-Filecoin storage, ERC-8004 attestation, Pydantic `TrainingRun` records).
+
+### Repository layout
+
+```
+mindxtrain/
+├── pyproject.toml                 # name="mindxtrain", py>=3.12, Apache-2.0
+├── LICENSE                        # Apache 2.0
+├── NOTICE                         # BANKON copyright + upstream MIT (Z.ai for GLM-5.1, Apache for Qwen3.5)
+├── LICENSE-MIT-upstream-glm51     # verbatim Z.ai MIT notice
+├── README.md
+├── CHANGELOG.md
+├── Containerfile                  # Podman, not Dockerfile
+├── compose.yaml                   # podman-compose
+├── mindxtrain/
+│   ├── __init__.py
+│   ├── config/                    # JSON config with ${ENV} interpolation, ml-intern style
+│   │   ├── train_default.json
+│   │   ├── eval_default.json
+│   │   ├── deploy_default.json
+│   │   └── schema.py              # Pydantic schemas
+│   ├── data/                      # data pipeline
+│   │   ├── curate.py              # source → raw
+│   │   ├── dedupe.py              # MinHash + exact-match
+│   │   ├── filter.py              # quality, language, toxicity
+│   │   ├── tokenize.py            # tokenizer-aware
+│   │   ├── pack.py                # sequence packing
+│   │   ├── synth.py               # synthetic data via GLM-5.1 / Qwen3.5
+│   │   └── verify.py              # hash + manifest
+│   ├── models/                    # model registry
+│   │   ├── registry.py            # ModelRegistry, register/get
+│   │   ├── chat_template.py       # ChatTemplate abstraction
+│   │   ├── glm51.py               # backend
+│   │   ├── qwen35.py              # backend
+│   │   ├── deepseek_v32.py        # backend
+│   │   ├── mistral3.py            # backend
+│   │   └── phi4_mini.py           # backend
+│   ├── train/                     # trainer core
+│   │   ├── sft.py                 # full SFT + LoRA + QLoRA
+│   │   ├── dpo.py                 # DPO via TRL
+│   │   ├── grpo.py                # GRPO via TRL
+│   │   ├── rlhf.py                # PPO via TRL
+│   │   ├── tool_use.py            # BFCL-style tool-trajectory training
+│   │   ├── distributed.py         # accelerate / FSDP / DeepSpeed config builders
+│   │   └── callbacks.py           # eval-during-training, checkpoint mgmt
+│   ├── eval/                      # eval harness
+│   │   ├── lighteval_adapter.py
+│   │   ├── inspect_ai_adapter.py
+│   │   ├── bfcl.py                # BFCL v3 / v4
+│   │   ├── persona_regression.py  # Codephreak voice tests
+│   │   ├── agenda_regression.py   # agenda-conditioning tests
+│   │   ├── tau_bench.py
+│   │   └── card.py                # auto model card from run
+│   ├── operator/                  # ml-intern-pattern-derived
+│   │   ├── tool_router.py         # ToolRouter + ToolSpec
+│   │   ├── agent_loop.py          # bounded ReAct, doom-loop
+│   │   ├── context.py             # 170k compaction
+│   │   ├── trajectory.py          # JSONL writer
+│   │   ├── approval.py            # CLI/web/Slack approval flow
+│   │   └── prompts/
+│   │       ├── system_v1.yaml     # Research → Plan → Implement
+│   │       └── codephreak.yaml    # persona + agenda, prompt-as-data
+│   ├── storage/                   # provenance layer
+│   │   ├── provider.py            # StorageProvider interface
+│   │   ├── local_fs.py            # always-available fallback
+│   │   ├── hf_hub.py
+│   │   ├── lighthouse.py          # Lighthouse Storage / Filecoin
+│   │   └── ipfs.py                # raw IPFS
+│   ├── provenance/                # blockchain anchoring
+│   │   ├── manifest.py            # Pydantic TrainingRun record
+│   │   ├── erc8004.py             # Identity + Reputation registry attestation
+│   │   ├── algorand.py            # BANKON ASA 203977300 hooks
+│   │   └── x402.py                # HTTP-402 micropayment for paid inference
+│   ├── deploy/                    # automindX v2 integration
+│   │   ├── registry.py            # model-version registry
+│   │   ├── hot_swap.py            # atomic swap with canary
+│   │   ├── ab_test.py             # A/B traffic split
+│   │   └── api_client.py          # mindx.pythai.net OpenAI-compat client
+│   ├── budget/                    # ResourceBudget (psutil-derived from aGLM)
+│   │   └── resource.py
+│   └── cli/                       # snake_case entries
+│       └── main.py                # mindxtrain.cli.main:cli
+├── contracts/                     # Foundry, for ERC-8004 hooks
+│   ├── foundry.toml
+│   ├── src/
+│   ├── test/
+│   └── script/
+├── ops/
+│   ├── containerfiles/            # Podman build files per role
+│   ├── compose/                   # podman-compose stacks
+│   ├── vmm/                       # OpenBSD vmm vm definitions
+│   └── gensyn/                    # Gensyn distributed-training configs
+├── tests/                         # pytest, persona regression
+├── docs/                          # .py↔.md colocated where reasonable
+└── scripts/                       # dev helpers
+```
+
+### Hardware feasibility — VRAM math worked out per target
+
+For **single H100 80 GB**, GLM-5.1 is impossible at any quantization (754 GB FP8 weights alone). Realistic targets are Qwen3-32B in BF16 (~64 GB weights + ~6 GB KV at 8 K + ~4 GB activations + grad + optimizer for inference, but training requires ZeRO offload), Qwen3-7B in BF16 with full LoRA (14 GB weights + LoRA adapters ~200 MB + Adam state ~28 GB for FP32 master + 14 GB grads — tight, use QLoRA), Phi-4-mini in BF16 with full SFT (~7.6 GB weights + 15.2 GB grads + 30.4 GB Adam states ≈ 53 GB, fits with margin), Gemma 4 31B with QLoRA. Active ~10B-class MoEs like Qwen3.5-35B-A3B fit in inference at INT4 (~17.5 GB weights + 8 GB KV + activations ≈ 30 GB), and QLoRA fine-tuning is feasible at 16K context.
+
+For **8× H100 80 GB cluster** (640 GB aggregate), GLM-5.1-FP8 is borderline — 754 GB weights does not fit even with ZeRO-3 splitting unless KV is compressed via MLA and you accept 16-bit gradient checkpointing with CPU offload (KTransformers-style); the realistic posture is "inference yes via FP8 MLA-compressed, training no." Qwen3-235B-A22B in BF16 fits trivially for inference (~470 GB weights) and is trainable via FSDP+ZeRO-3 with QLoRA (active 22 B → adapter math is reasonable). Qwen3.5-122B-A10B in BF16 (~244 GB) is comfortable for both inference and full SFT. Mistral Large 3 (675 B / 41 B-A) at FP8 is comparable to GLM-5.1 — borderline. The 8×H100 sweet spot for full SFT is in the 32B–122B-active range.
+
+For **single A100 80 GB**, treat as a slightly slower H100 with the same memory ceiling. GLM-5.1 still impossible. Qwen3-32B QLoRA feasible. Same VRAM math as H100.
+
+For **8× A100 80 GB**, identical capacity to 8× H100 (640 GB) but lower throughput; same model ceilings.
+
+For **RTX 4090 24 GB**, GLM-5.1 impossible. Qwen3-7B QLoRA feasible (4-bit weights ~3.5 GB + LoRA ~200 MB + KV at 4 K ~1 GB + grads + Adam ≈ 18 GB). Phi-4-mini full SFT feasible (BF16, ~16 GB total). Qwen3-1.7B and Qwen3-0.6B full SFT comfortable. Edge fine-tunes only.
+
+For **Apple Silicon M3 Ultra 192 GB unified memory** via MLX, GLM-5.1 impossible until `mlx-lm` issue #879 lands and INT4 quants become available (then INT4 + MLA-compressed KV ≈ 383 GB at 128K context — *still* doesn't fit). Qwen3-32B BF16 fits with margin (~64 GB + KV). Qwen3.5-122B-A10B in INT4 (~30 GB weights) fits comfortably for inference; QLoRA fine-tuning works via MLX-LM's PEFT support. Mistral Large 3 at INT4 borderline (~169 GB + KV). M2 Max 96 GB is more constrained: Qwen3-32B INT4 (~16 GB) + LoRA fine-tune is the realistic ceiling.
+
+For **Gensyn distributed training** (the \$5K hackathon track), the framework's posture is to use Gensyn's RL Swarm SDK over the WAN training fabric for the agentic-trajectory RL phase only — pretraining and SFT happen on owned/leased H100/A100 clusters; the Gensyn integration enters at the GRPO stage where many small rollout workers (Qwen3-7B / Phi-4-mini scale) generate trajectories that the central trainer aggregates. This both fits the Gensyn programming model and exercises the BANKON x402 micropayment rail when each rollout worker is paid per accepted trajectory.
+
+### Model size selection logic for mindX cognitive API
+
+The mindX cognitive API at `mindx.pythai.net` should serve a tiered family, not a single flagship. The recommended composition: **edge tier** is `aGLM-BANKON-edge` derived from Phi-4-mini (3.8 B, MIT, BFCL 70.3) for sub-second-latency tool-call routing and on-device deployments; **mid tier** is `aGLM-BANKON-mid` derived from Qwen3-7B or Qwen3.5-Flash for the bulk of agentic traffic where per-token cost matters; **flagship tier** is `aGLM-BANKON-flag` derived from Qwen3.5-122B-A10B for hard agentic flows where 22 B-class active capacity isn't enough but 41 B-class is overkill; **specialist tier** is `aGLM-BANKON-spec` derived from GLM-5.1-FP8 for the long-horizon SWE / 8-hour-autonomous-session use case where GLM-5.1's SWE-Bench Pro 58.4 dominance and 200K-context DSA are decisive. Routing between tiers happens at the operator layer based on task classification (tool-call routing → edge; chat with tools → mid; complex agentic flow → flagship; long-horizon SWE → specialist).
+
+### Data pipeline
+
+The dataset construction pipeline is a Pydantic-typed DAG: `curate` (pull from source — HF Datasets, Common Crawl-derived, codephreak conversation history, mindX session logs) produces `raw/`; `dedupe` runs MinHash near-duplicate detection plus exact-match removal producing `deduped/`; `filter` applies language detection (`fasttext`), quality classifiers (Cosmopedia-style), and toxicity filtering producing `filtered/`; `tokenize` runs the target tokenizer with vocab-hash recorded into the manifest producing `tokenized/`; `pack` does sequence packing to the target context length producing `packed/`; `synth` is the optional synthetic-data generation step that uses GLM-5.1 (specialist tier) or Qwen3.5-122B-A10B (flagship tier) to generate domain-specific tool-use trajectories, persona-conditioned dialogues, and edge-case examples (the ml-intern healthcare demo's "generate 1,100 synthetic edge cases and upsample 50×" pattern is the template); `verify` produces a manifest containing every artifact's BLAKE3 hash, the Lighthouse-Filecoin CID, the source URLs, and the generation parameters. The whole pipeline is a single `mindxtrain.data.run(config)` call that produces a `DatasetManifest` Pydantic record consumed by `train`.
+
+### Evaluation harness
+
+The eval harness has three sub-layers: standard benchmarks via `lighteval` and `inspect-ai` adapters (MMLU-Pro, GPQA-Diamond, AIME, MATH, HumanEval, MBPP, IFEval); agentic benchmarks via custom adapters (BFCL v3/v4 — full Berkeley harness wired in, τ²-Bench, τ³-Bench, AgentBench, GAIA, MCP-Atlas where dataset is public, SWE-Bench Verified — Pro is gated behind Scale AI submission); and the **persona-and-agenda regression suite**, which is unique to mindXtrain and irreplaceable. The persona suite tests "does the model still call itself codephreak", "does the model preserve the Professor Codephreak ML/CS/programming domain", "does the model stay agenda-conditioned when given a multi-step build task", "does the model emit the correct chat-template tokens for the target backend", and "does the model degrade gracefully when the agenda is unfulfillable." Regression detection is automatic: every checkpoint runs the full suite, scores are diffed against the baseline (the most recent green checkpoint), and any score regression beyond a configurable tolerance halts the deployment pipeline. Model card auto-generation reads the `TrainingRun` manifest plus eval JSONs and emits a Hugging Face-compatible `README.md` with full provenance.
+
+### Training pipeline configurations — copy-pasteable
+
+A canonical Qwen3.5-122B-A10B SFT-LoRA configuration in TRL/PEFT, expressible directly in the mindXtrain JSON config:
+
+```json
+{
+  "run_id": "aGLM-BANKON-flag-sft-001",
+  "base_model": {
+    "id": "Qwen/Qwen3.5-122B-A10B",
+    "revision": "main",
+    "license": "apache-2.0",
+    "vocab_hash": "blake3:..."
+  },
+  "tokenizer": {"chat_template": "qwen3"},
+  "dataset": {
+    "manifest_cid": "lighthouse://bafy...codephreak-sft-v3",
+    "format": "jsonl-chatml",
+    "max_seq_len": 16384,
+    "packing": true
+  },
+  "trainer": {
+    "type": "sft",
+    "framework": "trl",
+    "lora": {"r": 64, "alpha": 128, "dropout": 0.05,
+             "target_modules": ["q_proj","k_proj","v_proj","o_proj",
+                                "gate_proj","up_proj","down_proj"]},
+    "qlora": {"bits": 4, "compute_dtype": "bfloat16",
+              "double_quant": true, "quant_type": "nf4"}
+  },
+  "optim": {
+    "optimizer": "paged_adamw_32bit",
+    "lr": 2e-5, "lr_scheduler": "cosine", "warmup_ratio": 0.03,
+    "weight_decay": 0.0, "max_grad_norm": 1.0,
+    "epochs": 3, "global_batch": 64, "micro_batch": 1,
+    "grad_accum": 8, "grad_checkpointing": true
+  },
+  "distributed": {
+    "strategy": "fsdp", "shard": "full", "mixed_precision": "bf16",
+    "cpu_offload": false
+  },
+  "callbacks": {
+    "eval_steps": 200, "save_steps": 500,
+    "eval_suites": ["bfcl_v4","persona_regression","agenda_regression"],
+    "stop_on_regression": true
+  },
+  "storage": {"provider": "lighthouse",
+              "checkpoint_dir": "lighthouse://aGLM-BANKON-flag/sft-001/"},
+  "provenance": {"erc8004_attest": true,
+                 "x402_settlement": false,
+                 "algorand_asa": 203977300}
+}
+```
+
+For DPO, swap `trainer.type` to `dpo`, point the dataset at a preference manifest with `chosen`/`rejected` pairs, drop the LR to `5e-7`, set `beta: 0.1`, and target the same LoRA modules — TRL's `DPOTrainer` consumes this directly.
+
+For GRPO over agentic trajectories, `trainer.type: "grpo"` with `reward_funcs: ["bfcl_pass","tau_bench_score","persona_consistency"]`, `num_generations: 8`, `temperature: 0.9`, `max_prompt_length: 8192`, `max_completion_length: 8192`. The reward functions are pluggable Python callables registered in `mindxtrain.train.grpo.reward_registry`.
+
+## Part 5 — automindX v2 integration
+
+automindX v2 is the consumer of mindXtrain-produced `aGLM-BANKON-*` checkpoints. The upgrade path from `pythaiml/automindx/aglm.py` to v2 is mechanical given the gap analysis. The single class `LlamaModel(model_name, models_folder)` becomes the package `automindx.models.{Backend}` with backends `HfTransformersBackend`, `LlamaCppBackend`, `OllamaBackend`, `VllmBackend`, `OpenAiCompatBackend`, `ZaiBackend`, `AnthropicBackend`, `GroqBackend` — registered in `automindx.models.registry.ModelRegistry`, dispatched by string key, picking the right `automindx.chat.ChatTemplate` for the backend and falling back to `tokenizer.apply_chat_template`. The synchronous `generate_contextual_output` becomes `generate_contextual_output(context: ConversationContext) -> Generation`, with an `agenerate_contextual_output` async sibling — sync default surface is preserved so existing call sites work unchanged. The JSON-on-disk memory becomes a `MemoryStore` interface with backends `JsonFsBackend` (default, never break), `PgVectorScaleBackend`, `LighthouseFilecoinBackend`. The 4096-character ceiling (`chunk4096.py`) is replaced by tokenizer-aware truncation plus sliding-window summarization for long contexts; default ctx 8K–128K depending on backend. The hard-coded persona becomes `prompts/codephreak.yaml` with the persona, the agenda field as a first-class slot, and the chat-template-token mapping declared.
+
+The model registry, versioning, and hot-swap mechanics: `automindx.deploy.Registry` is a content-addressed registry where every `aGLM-BANKON-*` checkpoint is identified by the BLAKE3 hash of its safetensors plus its `TrainingRun` manifest CID. The registry tracks `current`, `canary`, and `rollback` pointers per tier (edge, mid, flagship, specialist). Hot-swap is atomic: `automindx.deploy.HotSwap.promote(tier, run_id)` flips the `current` pointer after running the persona-regression and agenda-regression suites against the live API harness; on failure, it auto-rollbacks. A/B testing is a traffic split at the `automindx.api.Router` layer: 95% to `current`, 5% to `canary`, with both branches logged as Claude-Code-JSONL trajectories to Lighthouse for offline statistical comparison via `automindx.eval.ab_compare(run_a, run_b, metric)`. The mindx.pythai.net public API exposes an OpenAI-compatible `/v1/chat/completions` plus a mindX-native `/v1/agentic` that takes an agenda field directly and returns a session ID plus streaming events. Every accepted request is provenance-linked: the model checkpoint hash, the run-id, the registry's ERC-8004 attestation, and (for paid tiers) the x402 micropayment receipt are all included in response headers.
+
+## Part 6 — comparative due diligence: the verdict
+
+The brief asked for a comparison of GLM-5.1 against Qwen3 (235B / 72B / 32B / 14B / 7B / 4B / 1.7B / 0.6B), Llama 3.3 70B, DeepSeek V3.1, Mistral Large 2, Gemma 3, and Phi-4. The May-2026 reality has moved beyond several of those: Qwen3.5 (Flash, 27B-dense, 35B-A3B, 122B-A10B) ships and outperforms Qwen3-235B on multiple axes; Llama 4 (Scout/Maverick, April 2025) ships under the *Llama 4 Community License* with the 700M MAU clause and EU AUP; DeepSeek V3.2 / V3.2-Speciale (December 2025) ships under MIT; Mistral Large 3 + Ministral 3 (December 2025) ships under Apache 2.0; Gemma 4 (April 2026) ships under Apache 2.0 — the single biggest license unlock of 2026; Phi-4 / Phi-4-mini ships under MIT.
+
+Apache-2.0 redistribution requires the upstream to be permissive: Apache 2.0, MIT, or BSD-class. That filter cleanly admits **GLM-5.1, Qwen3, Qwen3.5, DeepSeek V3.2, Mistral Large 3 + Ministral 3 + Mistral Small 4, Gemma 4, Phi-4, Phi-4-mini, and Yi-1.5**, and cleanly rejects **Llama 3.3, Llama 4, Cohere Command A** (CC-BY-NC, non-commercial), **Falcon 3** (TII Falcon 2.0 with AUP — most counsel will flag this as restrictive), and **Yi-Lightning** (proprietary API-only). That kills five of the ten brief candidates before benchmarks are scored.
+
+The verdict, on the dimensions of license-compatibility, agentic capability, tool-use quality, fine-tunability, ecosystem support, performance-per-parameter, multilingual balance, and size-ladder coverage, is that **mindXtrain should target Qwen3.5 as the primary base and treat GLM-5.1 as a premium specialist track**. Six reasons.
+
+First, license strength is symmetric. MIT and Apache 2.0 are functionally equivalent for downstream Apache 2.0 redistribution; both pass cleanly. There is no license-based reason to prefer GLM-5.1.
+
+Second, GLM-5.1's wins are real but narrow. SWE-Bench Pro 58.4 is genuinely SOTA. Terminal-Bench 2.0, MCP-Atlas, BrowseComp, CyberGym, AIME 2026 95.3, GPQA-Diamond 86.2 are all top-tier. But for tool-use plus function-calling plus web automation — the brief's stated agentic spec — GLM-5.1 is overkill. The 8-hour-autonomous-session capability is for software-engineering agents specifically.
+
+Third, fine-tunability gap. GLM-5.1 was 27 days old at research time; Axolotl recipes are still maturing, LLaMA-Factory support is partial, Unsloth has GGUF but full LoRA/QLoRA flows are not battle-tested. Qwen3 and Qwen3.5 have day-one PEFT, Axolotl, LLaMA-Factory, Unsloth, and TRL support with hundreds of teams' worth of exercised DPO and GRPO recipes.
+
+Fourth, per-parameter economics. GLM-5.1 at 754B/40B-A requires 4–8×H100 to *serve* and ZeRO-3 territory to fine-tune. Qwen3.5-122B-A10B has roughly comparable intelligence at one-quarter the active footprint and runs on 2×H100 comfortably; Qwen3.5-35B-A3B with 3B active reportedly exceeds Qwen3-235B-A22B. The agentic fine-tune sweet spot for mindXtrain is the 30B–122B-active class.
+
+Fifth, size ladder. mindXtrain needs a *family*, not a single model — flagship for hard flows, mid-tier for production traffic, edge for cost-or-latency-sensitive deployments. Only Qwen has a contiguous Apache-2.0 family from 0.6B → 235B → 235B-Instruct-2507 → 235B-Thinking-2507 → 3.5 medium series. GLM-5.1 is one model. Family completeness alone tips this toward Qwen.
+
+Sixth, BFCL leadership lives in the Qwen lineage. Qwen3-32B hits BFCL v3 75.7%; Qwen3.5-122B-A10B reaches BFCL v4 72.2%; the GLM-4.5 ancestor leads at 76.7%, but GLM-5.1 has not officially submitted to BFCL. For a function-calling-first agentic framework, Qwen has more *demonstrated* surface.
+
+The recommendation, restated in concrete pin-the-version-and-go form: target Qwen3.5-Flash for the edge tier or Phi-4-mini if BFCL-at-tiny-size matters more than open-Apache-only purity (Phi-4-mini is MIT, equivalent for redistribution); target Qwen3-7B for mid-tier; target Qwen3.5-122B-A10B for flagship; maintain GLM-5.1-FP8 as the specialist track for long-horizon SWE flows where 8-hour autonomous sessions and SWE-Bench Pro 58.4 are decisive; keep DeepSeek V3.2 as the reasoning-with-tool-use track backup if Qwen3.5-Thinking-class falls short on internal evals; keep Mistral Large 3 + Ministral 3 as the EU-jurisdictional secondary track; keep Gemma 4 as a watch-list option once vLLM kernel optimization and Axolotl recipes mature (the early-release speed and stability issues should clear within 60–90 days); do not base anything on Llama 3.3, Llama 4, Cohere Command A, Falcon 3, or Yi-Lightning regardless of benchmark performance.
+
+## Conclusion: provenance chain and execution order
+
+The end-to-end provenance chain is: **upstream open-weight base** (Qwen3.5-122B-A10B Apache 2.0 for primary, GLM-5.1-FP8 MIT for specialist) → **mindXtrain training pipeline** (data DAG with Lighthouse-anchored manifests, SFT + LoRA + DPO + GRPO + tool-use trajectory training, full eval harness with persona regression) → **`aGLM-BANKON-*` checkpoint** (Apache 2.0, NOTICE preserves upstream MIT / Apache, BLAKE3-hashed and Lighthouse-Filecoin-CIDed) → **ERC-8004 attestation** (Identity Registry `0x8004A169...` and Reputation Registry `0x8004BAa1...` on EVM via Foundry-tested contract calls, BANKON ASA `203977300` on Algorand for settlement) → **automindX v2 model registry** (content-addressed, hot-swap with canary and rollback, persona-and-agenda regression gates) → **mindx.pythai.net cognitive API** (OpenAI-compatible `/v1/chat/completions` plus mindX-native `/v1/agentic`, x402 micropayment for paid tiers, full provenance in response headers).
+
+Execution order, the boring critical-path version: first, lock the license posture — pull `zai-org/GLM-5.1-FP8` to a self-hosted store, write the Apache-2.0 + MIT-NOTICE compliance bundle for the BANKON derivative, draft the `mindx.pythai.net` Terms of Service that does *not* inherit Z.ai's AUP. Second, stand up the Qwen3.5-122B-A10B fine-tuning rail end-to-end on whatever 8×H100 / 8×A100 capacity is available, validate it with a deliberately-trivial Codephreak-persona SFT to exercise every callback, every storage backend, every regression gate. Third, port `pythaiml/automindx/aglm.py` to `automindx.models.HfTransformersBackend` with the `ChatTemplate` abstraction, preserving the persona-as-agenda discipline and the four-axis decomposition, in a v2 branch that the v1 callers can opt into without breaking. Fourth, reimplement the five ml-intern operator patterns into `mindxtrain.operator.*` once HF posts the LICENSE on `huggingface/ml-intern` (issue #41), or right now if the patterns are clean-room reimplemented from the public API description without copying source. Fifth, wire the storage and provenance layers — Lighthouse-Filecoin first (CIDs anchored in `TrainingRun` manifests), ERC-8004 attestation second (Foundry-tested contract calls from `mindxtrain.provenance.erc8004`), x402 settlement last. Sixth, take the resulting `aGLM-BANKON-flag-001` to the Gensyn distributed-RL track for the GRPO trajectory phase, exercising the WAN training fabric and the x402 per-trajectory micropayment rail simultaneously.
+
+The framing that matters for everything downstream: GLM-5.1 is a remarkable model, the strongest open-weights agentic-engineering base in May 2026, and a serious specialist track for mindXtrain. But it is not, on the rigorous evidence, the right *primary* base for a multi-tier Apache-2.0-redistributable agentic family. The right primary is Qwen3.5. Building the framework with that priority pair locked in — Qwen3.5 primary, GLM-5.1 specialist — is what makes mindXtrain a production-grade training framework rather than a single-model wrapper, and what gives `aGLM-BANKON` the family completeness it needs to serve every cell of the mindX cognitive API at `mindx.pythai.net`.

docs/blueprints/mindXtrain_ Production Blueprint for the AMD and lablab.ai Hackathon.md

@@ -0,0 +1,499 @@
+# mindXtrain — production blueprint for the AMD × lablab.ai hackathon (May 4–10 2026)
+
+**The clock is already running.** The lablab.ai AMD Developer Hackathon opened its online build window on May 4 2026, the on-site finale runs May 9–10 in San Francisco at the MindsDB SF AI Collective, and the prize pool is **$21,500+ plus an AMD Radeon AI PRO R9700 GPU** across three tracks (Agents, Fine-Tuning on AMD GPUs, Vision/Multimodal) with stackable cross-cutting prizes for **Best Use of Qwen** and **Build in Public**, and a Hugging Face Spaces "most likes" prize topped by a Reachy Mini Wireless robot. mindXtrain enters as the **Fine-Tuning track** primary, cross-submitted to Best-Use-of-Qwen and Build-in-Public, with a one-line pitch nobody else in the live `lablab-ai-amd-developer-hackathon` HF org currently occupies: *"the first one-command Qwen3.6 fine-tuner natively optimized for MI300X — auto-selects Composable Kernel, AITER, hipBLASLt and Flash-Attention-ROCm configs from a 60-second micro-benchmark, trains via Optimum-AMD + TRL, quantizes with AMD Quark, and serves on vLLM-ROCm 7.2.1, all from a single CLI."* That positioning hits all four explicit lablab judging criteria — Technology Integration, Presentation, Business Value, Originality — and it directly maps onto AMD's actual KPI of ROCm developer adoption, which matters because the judge bench is led by **Ramine Rozen, CVP of AI at AMD**. The remainder of this document is the production blueprint.
+
+## How the hackathon actually scores, and the angle that wins it
+
+The lablab page lists four criteria with no explicit weights; sister AMD events (Bengaluru, Delhi) reveal a strong implicit theme of **production-readiness under hardware constraints** — judges reward submissions that exploit MI300X's 192 GB HBM3 to do something an H100 80 GB cannot. The current bar to beat is **REPOMIND**, a repo-scale coding agent in the live HF org built on Qwen3-Coder-Next-FP8 + vLLM ROCm 7 that explicitly markets "H100 OOMs on this workload, MI300X just runs it." mindXtrain's differentiator is orthogonal — it is *infrastructure that produces specialized Qwen3 models cheaply on MI300X*, not a single specialized application — and the AutoML/AutoTrain niche is genuinely empty in the lablab archive. The submission therefore needs to be **demonstrable in 90 seconds**, must produce a tangible artifact (a finetuned Qwen3 checkpoint pushed to the HF org, served on a public Space), and must show the AMD stack being *fully* exercised at every layer rather than treated as a black box. The Build-in-Public extension costs almost nothing — two technical X posts tagging @lablab and @AIatAMD plus an MIT-licensed repo are required anyway — so it is a free additional prize pool.
+
+The **single most differentiating angle** is the auto-selection layer. No competitor framework — not Axolotl, LLaMA-Factory, Unsloth, torchtune, Primus, or Optimum-AMD itself — runs a **per-job MI300X micro-benchmark** before training to pick CK vs Triton attention backends, hipBLASLt heuristic vs rocBLAS path, AITER vs reference MoE kernels, NCCL_MIN_NCHANNELS, gradient-checkpointing strategy, FSDP shard width, and LoRA rank against the actual (model, dataset shape, sequence length, GPU count) tuple. mindXtrain owns that AOT-only autotune layer.
+
+## The mindXtrain reference stack, pinned
+
+The recommended container is **`rocm/primus:v26.2`** (which is identical to `rocm/pytorch-training:v26.2` and bundles ROCm 7.2.1, PyTorch 2.9.1, AOTriton 0.11+, AITER 0.1.12, RCCL with Pollara fixes, Apex with fused RoPE, Megatron-Core, TorchTitan, Primus-Turbo, ROCm/TransformerEngine, and Quark). Pull it once, snapshot the SHA256 digest into `infra/podman/digest.lock`, and never run training off a floating tag. For lighter SFT-only flavors a thinner image — **`rocm/pytorch:rocm7.2.1_ubuntu24.04_py3.12_pytorch_release_2.9.1`** — is acceptable.
+
+Above the container, the layered Python stack is `torch==2.9.1+rocm7.2.1.lw` (from `repo.radeon.com/rocm/manylinux/rocm-rel-7.2.1`, *not* `download.pytorch.org/whl/nightly/rocm7.2`, because the AMD-validated wheels are reproducible while nightlies churn), `triton==3.5.1+rocm7.2.1`, `transformers>=4.46,<4.50`, `accelerate>=1.0`, `peft>=0.13`, `trl>=0.12`, `datasets>=3.0`, `optimum>=1.24`, `optimum-amd` from main, `amd-quark>=0.11.1`, `auto-gptq` from `huggingface.github.io/autogptq-index/whl/rocm573`, `bitsandbytes` built from `github.com/ROCm/bitsandbytes` branch `rocm_enabled_multi_backend` with `-DBNB_ROCM_ARCH="gfx942;gfx950" -DCOMPUTE_BACKEND=hip`, `flash-attn` built from `Dao-AILab/flash-attention` upstream with both CK and Triton backends compiled (`FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE`), and `vllm` ROCm 7.0+ wheels for serving. **Numpy must be pinned `<2.0`** against torch 2.9 ROCm wheels.
+
+The runtime environment is non-negotiable on MI300X: `HSA_NO_SCRATCH_RECLAIM=1`, `NVTE_CK_USES_BWD_V3=1`, `NVTE_CK_IS_V3_ATOMIC_FP32=1`, `PRIMUS_TURBO_ATTN_V3_ATOMIC_FP32=1`, `HIP_FORCE_DEV_KERNARG=1`, `PYTORCH_ROCM_ARCH=gfx942`, `NCCL_MIN_NCHANNELS=112` for sub-8-GPU jobs, `GPU_MAX_HW_QUEUES=1` for multi-GPU stability, and `numa_balancing` off in `/proc/sys/kernel`. **Pin to ROCm 7.2.1 or later** — 7.1.x has a documented bf16/fp16 GEMM cu-fallback regression on gfx942 that costs ~600 ms per call through hipBLASLt heuristic scans. The deprecated `ROCm/Megatron-LM` Docker is being retired in favor of Primus; adopt Primus from day zero to avoid migration debt.
+
+## The Qwen3 family targeting decision
+
+**Qwen3.6 is real**, contrary to the user's implicit doubt. The open-weight cadence reads Qwen3 (April 2025, original 8 dense + 2 MoE checkpoints) → Qwen3-2507 split refresh (July 2025, 256K native context) → Qwen3-Next-80B-A3B (September 2025, hybrid Gated DeltaNet + sparse MoE) → Qwen3-Coder/VL/Omni/Guard/ASR/Image (Aug–Oct 2025) → Qwen3.5 family (Feb 16 2026, unified text+vision backbone, flagship 397B-A17B) → **Qwen3.6** (April 2026, currently the newest open-weight checkpoint as of May 5 2026). The two open Qwen3.6 checkpoints are `Qwen/Qwen3.6-27B` (dense, released April 22 2026) and `Qwen/Qwen3.6-35B-A3B` (MoE, released April 16 2026), both Apache 2.0, both multimodal `image-text-to-text`, both 262,144 native context extendable to ~1M via YaRN, both default-thinking with no `/think` `/no_think` soft switch (use `chat_template_kwargs={"enable_thinking": False}` instead), and both introduce **Thinking Preservation** (`preserve_thinking=True`) for agentic multi-turn KV-cache reuse. Vocab grew from 151,936 in Qwen3 to 248,320 in Qwen3.6 to support 201 languages.
+
+mindXtrain's **default targets** for the hackathon demo should be a tier of three: **Qwen3-8B** as the headline single-MI300X full-FT case (fits one card with bs=8 seq=4096, AdamW, bf16, ~80 GB peak, 12–20k tok/s on Primus-Turbo); **Qwen3-32B** as the FSDP-2 4×MI300X full-FT showpiece (bs=2, seq=4096, ~5–9 hours per 1B tokens at FP8 with TE-CK); and **Qwen3.6-35B-A3B** as the latest-and-greatest MoE LoRA case (experts-only adapters, gate frozen, 2×MI300X with EP enabled). Qwen3-235B-A22B-Thinking-2507 is the dramatic-but-impractical option — LoRA on FP8/MXFP4-quantized weights across 8×MI300X is feasible at ~5–12k tok/s; full FT requires multi-node and is out of hackathon scope.
+
+Two Qwen3-specific landmines deserve naming. First, hybrid Gated DeltaNet layers in Qwen3-Next, Qwen3.5 and Qwen3.6 do not use standard SDPA — they require `flash-linear-attention` and `causal-conv1d`, and on ROCm both need community wheels or source builds; without them you fall back to a slow PyTorch reference path. Second, the **MoE router gate** must be frozen during fine-tuning; thawing it almost always diverges. Both pitfalls belong in mindXtrain's autotune as hard rules, not user-facing knobs.
+
+The Qwen team's preferred RL algorithm is **GRPO** (Qwen3 launch/2507) and its successor **GSPO** (Qwen3-Next/3.5/3.6, recommended for hybrid + sparse MoE stability). AMD has published an end-to-end ROCm + TRL + vLLM + DeepSpeed GRPO recipe on MI300X using GSM8K and Qwen2.5-1.5B-Instruct — that recipe transposes one-to-one onto Qwen3-8B and is the safest single-node demo path.
+
+## What the open-source landscape gives us, and what it doesn't
+
+The framework survey produced a clear ranking. **Axolotl** (Apache-2.0, YAML-driven, 70k+ models, March 2026 added Qwen3.5/Qwen3.5-MoE, ND parallelism, FP8 via torchao) is the right *YAML skeleton* for mindXtrain's multi-GPU SFT/DPO/ORPO/GRPO/QAT path; ROCm support is second-class but the AI-DarwinLabs `amd-support` branch ships a working `requirements-amd.txt` and AMD itself published a Dockerfile.rocm walkthrough. **LLaMA-Factory** (Apache-2.0, 70.6k stars, Qwen team's own preferred trainer, AMD's own MI300X tutorial uses it) is the right *Qwen3-specific recipe library* — supports every Qwen3 variant including 2507, VL, Omni and Next, plus PPO/DPO/KTO/ORPO/SimPO/GRPO. **torchtune** (BSD-3, AMD-CI-tested, the cleanest codebase) is the right *modular reference implementation*, weak only because Qwen3 recipes are not yet upstream — a ~200-line PR that mindXtrain should ship as a side-deliverable. **Unsloth** (Apache-2.0, official AMD partnership via OneClickAMD, fastest single-GPU experience) belongs as an **opt-in backend** for single-MI300X jobs where its 16-bit LoRA path beats everyone else, with the caveat that bitsandbytes 4-bit is currently unstable on ROCm 7.x and Unsloth correctly disables it. Below those four sit **TRL + PEFT + Accelerate** as mandatory dependencies (every higher-level trainer wraps them; Accelerate is first-class on ROCm without code changes), **Optimum-AMD** as the mandatory Flash-Attention-2/GPTQ/`amdrun`-topology shim (now MIT-licensed, Apache-compatible), **DeepSpeed** as the first-class ZeRO/MoE backend on ROCm 6+, **vLLM-ROCm** and **SGLang** as first-class rollout engines for GRPO and serving (vLLM ROCm CI went live December 2025, SGLang ships explicit `rocm/sgl-dev:v0.5.8.post1-rocm720-mi30x` images), and **AMD-AGI/Primus + Primus-Turbo** as the right pretraining-scale fallback when mindXtrain is asked to do continued pretraining at >70 B parameters. NeMo is disqualified (CUDA-only TransformerEngine dependency), the AMD GPT-NeoX branch is three years stale, Megatron-DeepSpeed is superseded, and Lamini is CC-BY-NC and disqualified by license.
+
+The whitespace mindXtrain fills is precisely the seven gaps no existing framework owns: **(1)** MI300X-aware automated hyper-parameter selection driven by a 60-second micro-benchmark; **(2)** a one-line install matrix that pins a known-working ROCm/PyTorch/Flash-Attn/bitsandbytes/Unsloth tuple per ROCm version; **(3)** Qwen3-family torchtune recipes (upstreamable PR); **(4)** a unified `mindxtrain.yaml` schema that compiles down to either Axolotl or Unsloth backends based on cost/scale; **(5)** a native MI300X 4-bit path that bypasses bitsandbytes via Quark FP8/MXFP4 + LoRA-on-quantized weights; **(6)** auto-orchestration of vLLM-ROCm/SGLang rollout engines on idle GPUs of the same MI300X node for in-the-loop GRPO; and **(7)** a hackathon-grade reproducibility manifest — a "training receipt" capturing ROCm version, gfx target, every git SHA, the Docker digest, the exact YAML, and dataset hashes.
+
+## mindXtrain architecture
+
+mindXtrain is structured as five concentric layers. The **CLI layer** (`mindxtrain/cli.py`) exposes `mindxtrain init`, `mindxtrain bench` (the autotune micro-benchmark), `mindxtrain dataset prep`, `mindxtrain train`, `mindxtrain eval`, `mindxtrain quantize`, `mindxtrain serve`, `mindxtrain publish` and `mindxtrain receipt`. Each command consumes a single Pydantic-validated YAML config that supersedes Axolotl's, LLaMA-Factory's and torchtune's schemas; mindXtrain's job is to compile that config down to whichever backend YAML the underlying trainer expects. The **autotune layer** runs first — it inspects the (model, dataset, seq-len, GPU topology) tuple, executes a short profiled forward+backward to measure attention kernel throughput, GEMM heuristics, AllReduce bandwidth and HBM usage, then writes a `mindxtrain.tuned.yaml` with backend-specific overrides (CK vs Triton attention, AITER MoE on/off, NCCL_MIN_NCHANNELS, FSDP shard, LoRA rank ceiling, gradient-checkpointing policy). This layer is **AOT-only**: the autotune produces a static plan that is loaded at training start; **JIT autotune is forbidden in production** to keep training reproducible, in line with the user's existing mindX AutoTune discipline. The **dataset layer** wraps `datasets`, deduplicates with MinHash + SemDeDup, applies quality filtering, packs sequences (pack-to-cutoff Qwen3-style), shards for FSDP, and emits content-addressed Lighthouse Storage CIDs for provenance. The **training layer** dispatches to one of four backends — `axolotl` (multi-GPU SFT/DPO/ORPO/GRPO), `unsloth` (single-MI300X fast LoRA/GRPO), `torchtune` (modular recipes), or `primus` (pretraining-scale Megatron/TorchTitan) — selected by a `backend:` key with a sensible default chosen by autotune. Accelerate is the launcher in all cases except primus. The **artifact + integration layer** quantizes via Quark (FP8 / MXFP4 / INT4-FP8 two-level), evaluates with `lm-evaluation-harness` plus user-supplied custom evals with automatic regression detection against a baseline checkpoint, generates a model card, pushes to the HF org, mirrors the safetensors to Lighthouse Storage with a CID receipt, registers the trained model with the **mindX cognitive API** (`mindx.pythai.net`) as a new agent capability, lists it on **AgenticPlace** (`agenticplace.pythai.net`), allocates an **ENS subname** under `bankon.eth` for the resulting agent, and wires **x402 Algorand micropayments via parsec/parsec-wallet** for the training-as-a-service billing flow. Optional decentralized compute hooks for Bacalhau, Akash, and io.net are stubbed in `mindxtrain/compute/` so a job YAML can opt into running on a non-AMD-Cloud provider.
+
+The **CLI surface** is deliberately small. `mindxtrain init <project>` scaffolds a flat snake_case directory and a starter YAML. `mindxtrain bench --model Qwen/Qwen3-8B --hardware mi300x --gpus 1` runs the autotune. `mindxtrain train -c config.yaml` runs the full pipeline. `mindxtrain serve --checkpoint ./out/last --backend vllm-rocm` spins a vLLM-ROCm endpoint with the right tool-call and reasoning parsers (`hermes` + `qwen3`/`deepseek_r1` for non-coder Qwen3 models, `qwen3_coder` for Coder variants). `mindxtrain publish` pushes everything — checkpoint, model card, eval report, training receipt, Lighthouse CID — to HF, mindX, AgenticPlace and the BANKON ENS layer in one call. `mindxtrain receipt <run-id>` re-emits the manifest for any past run.
+
+The **config schema** is a single YAML with five top-level sections — `meta`, `model`, `data`, `train`, `serve` — each strongly typed by Pydantic. A canonical Qwen3-8B SFT-with-LoRA-on-1×MI300X config is reproduced below in the snippets section. Cross-cutting fields like `seed`, `provenance.lighthouse_endpoint`, and `hardware.gfx_arch` apply globally. Each training method (full FT, LoRA, QLoRA, DPO, ORPO, GRPO, GSPO, KTO, continued pretraining, multimodal Qwen3-VL/Omni/Image) is a discriminated union under `train.method`. **Hyperparameter search** uses Optuna by default (TPE sampler, MedianPruner) with the search space bounded by autotune-derived hard limits; Ray Tune is supported when a Kubernetes cluster is available; W&B Sweeps is the third option for teams already on W&B. The bounding rule is non-negotiable: autotune's profiled max batch size is the upper bound on `per_device_train_batch_size`, full stop.
+
+## Hackathon-winning submission strategy
+
+The 90-second demo storyline is concrete. Open with terminal: a single `mindxtrain train -c demo_qwen3_8b_sft.yaml` command. Cut to the autotune dashboard streaming MI300X micro-benchmark output (CK FA forward TFLOPS, hipBLASLt heuristic times, RCCL bus bandwidth) for 60 seconds. Cut to the training loop streaming loss + tok/s + MFU (target >40% MFU on Qwen3-8B BF16 with Primus-Turbo, comparable to AMD's published Llama-3.1-8B numbers). Cut to a side-by-side cost slide: **MI300X at $1.99/hr × 1 GPU × 3 hours = ~$6 versus H100 at $4/hr × 2 GPUs × 4 hours = ~$32**, with the headline "4× cheaper, can't OOM at 192 GB." Cut to the resulting model live in the HF Space chatting in thinking mode, then show `mindxtrain receipt` printing the full provenance manifest. Closing slide is one diagram of mindXtrain → mindX → AgenticPlace → BANKON. The story takes 90 seconds, leaves 60–90 seconds for Q&A in the 3-minute video budget.
+
+The deliverables checklist mapped to lablab's submission rubric is: a working prototype deployed as a Hugging Face Space inside the `lablab-ai-amd-developer-hackathon` HF org (Space name: `mindxtrain-demo`); a 2-3 minute demo video posted to YouTube and tweeted from the codephreak account tagging @lablab @AIatAMD @AMDROCm @huggingface @Alibaba_Qwen (auto-qualifies for Build-in-Public); a pitch deck of 5-7 slides Sequoia-style; the GitHub repo at `pythai/mindxtrain` MIT-licensed (or Apache-2.0 — verify the lablab page's "MIT-only" requirement on the live form, with Apache-2.0 as the codephreak-preferred fallback that may need a license-compat note); the lablab platform submission form filled with a 50-char title `mindXtrain — one-command Qwen3 on MI300X`, a 255-char short description, a 100-word-minimum long description naming every AMD library used, the cover image, the video URL, the GitHub URL, and the demo Space URL; verification of AMD AI Developer Program membership for the $100 cloud credits; and at least two technical X posts during the build window plus a written ROCm feedback note for Build-in-Public eligibility.
+
+The README structure for the GitHub repo is `# mindXtrain` headline, one-line tagline, a 30-second quick-start (`pipx install mindxtrain && mindxtrain init demo && mindxtrain train`), an architecture diagram (Mermaid), a benchmark table comparing mindXtrain to Axolotl/LLaMA-Factory/Unsloth/torchtune on Qwen3-8B-on-1×MI300X for the metrics tok/s, time-to-eval-loss-1.5, MFU, and total $; an "AMD stack exploited" section enumerating ROCm 7.2.1, AOTriton, AITER, Composable Kernel, hipBLASLt with offline tuning, Flash-Attention-CK, RCCL, Optimum-AMD, Quark, Primus-Turbo, vLLM-ROCm, and SGLang with one-line citations; a "what makes this MI300X-native" section pointing at the autotune; a license header; a roadmap; and an acknowledgments block thanking AMD, Hugging Face, and the Qwen team. The benchmark numbers to chase on the hero workload (Qwen3-8B SFT, 1×MI300X, bs=8, seq=4096, BF16, AdamW, 1B tokens) are **>15k tok/s, MFU >40%, time-to-loss-1.5 <90 minutes, total cost <$3**. Hit those and the cost slide writes itself.
+
+The comparison table mindXtrain prints in its README is the differentiator artifact: rows are Axolotl, LLaMA-Factory, Unsloth, torchtune, Primus, mindXtrain; columns are *one-command install on ROCm 7.2.1, MI300X auto-tune, Qwen3.6 day-zero recipe, FP8 via Quark, x402 micropayments, decentralized fallback, training receipt manifest*. mindXtrain is the only row with all seven cells filled.
+
+The risks and mitigations are well-defined. **Risk**: bitsandbytes 4-bit instability on ROCm 7.x. **Mitigation**: ship the Quark FP8 + LoRA path as default; bitsandbytes only as opt-in. **Risk**: Qwen3.6 GDN layers need flash-linear-attention/causal-conv1d which lack official ROCm wheels. **Mitigation**: build wheels in CI, ship in the Podman image, pin commit SHAs; fall back to PyTorch reference if build fails. **Risk**: lablab requires MIT license per submission rules but codephreak prefers Apache 2.0. **Mitigation**: dual-license the submission repo MIT for hackathon compliance, with a NOTICE file pointing at the upstream Apache-2.0 mindX ecosystem; reconcile post-hackathon. **Risk**: 60-second autotune window may not cover MoE expert imbalance. **Mitigation**: emit a warning in the receipt and run a longer autotune for Qwen3-30B-A3B / Qwen3.6-35B-A3B / Qwen3-235B-A22B specifically. **Risk**: vLLM-ROCm Triton autotune can stall first-batch on cold start. **Mitigation**: warm-up batch in `mindxtrain serve` before exposing the endpoint; or fall back via `VLLM_USE_TRITON_FLASH_ATTN=0`.
+
+## Repository skeleton
+
+The cypherpunk2048 standard is flat snake_case throughout, Apache-2.0 license header on every file, no proprietary lock-in, no upgradeable proxies in any Solidity, no EOA admin keys, Foundry for Solidity build, Podman over Docker:
+
+```
+mindxtrain/
+├── pyproject.toml
+├── README.md
+├── LICENSE                          # Apache-2.0 (hackathon submission may dual-license MIT)
+├── NOTICE
+├── mindxtrain/
+│   ├── __init__.py
+│   ├── cli.py                       # entry point (typer-based)
+│   ├── config.py                    # Pydantic schema
+│   ├── autotune/
+│   │   ├── __init__.py
+│   │   ├── benchmark.py             # 60-second MI300X probe
+│   │   ├── attention_probe.py       # CK vs Triton vs AITER selection
+│   │   ├── gemm_probe.py            # hipBLASLt offline tune trigger
+│   │   ├── rccl_probe.py            # bus-bandwidth measurement
+│   │   └── plan.py                  # writes mindxtrain.tuned.yaml (AOT)
+│   ├── dataset/
+│   │   ├── load.py
+│   │   ├── dedupe_minhash.py
+│   │   ├── dedupe_semdedup.py
+│   │   ├── quality_filter.py
+│   │   ├── tokenize.py
+│   │   ├── pack.py
+│   │   └── shard.py
+│   ├── train/
+│   │   ├── dispatch.py              # picks backend
+│   │   ├── backend_axolotl.py
+│   │   ├── backend_unsloth.py
+│   │   ├── backend_torchtune.py
+│   │   ├── backend_primus.py
+│   │   └── recipes/
+│   │       ├── qwen3_8b_sft_lora.yaml
+│   │       ├── qwen3_8b_sft_full.yaml
+│   │       ├── qwen3_32b_full_fsdp.yaml
+│   │       ├── qwen3_32b_dpo.yaml
+│   │       ├── qwen3_32b_orpo.yaml
+│   │       ├── qwen3_32b_grpo.yaml
+│   │       ├── qwen3_30b_a3b_lora.yaml
+│   │       ├── qwen3_6_27b_lora.yaml
+│   │       ├── qwen3_6_35b_a3b_lora.yaml
+│   │       ├── qwen3_vl_8b_sft.yaml
+│   │       └── qwen3_8b_cpt.yaml
+│   ├── eval/
+│   │   ├── harness.py               # lm-evaluation-harness wrapper
+│   │   ├── custom_eval.py
+│   │   └── regression.py
+│   ├── quantize/
+│   │   ├── quark_fp8.py
+│   │   ├── quark_mxfp4.py
+│   │   └── gptq_rocm.py
+│   ├── serve/
+│   │   ├── vllm_rocm.py
+│   │   ├── sglang_rocm.py
+│   │   └── parsers.py               # hermes / qwen3_coder / qwen3 reasoning
+│   ├── publish/
+│   │   ├── hf_hub.py
+│   │   ├── lighthouse.py            # IPFS / Filecoin via Lighthouse Storage
+│   │   ├── mindx_register.py        # POST to mindx.pythai.net
+│   │   ├── agenticplace_list.py     # POST to agenticplace.pythai.net
+│   │   └── bankon_ens.py            # ENS subname under bankon.eth
+│   ├── billing/
+│   │   ├── x402_algorand.py         # parsec/parsec-wallet integration
+│   │   └── pricing.py
+│   ├── compute/
+│   │   ├── amd_dev_cloud.py
+│   │   ├── tensorwave.py
+│   │   ├── bacalhau.py              # optional decentralized
+│   │   ├── akash.py
+│   │   └── ionet.py
+│   ├── telemetry/
+│   │   ├── prometheus_exporter.py
+│   │   ├── otel_hooks.py
+│   │   └── energy.py                # MI300X power tracking via rocm-smi
+│   ├── receipt/
+│   │   ├── manifest.py              # full provenance record
+│   │   └── verify.py
+│   └── chain_map/
+│       └── allchain.py              # consumes agenticplace.pythai.net/allchain.html
+├── infra/
+│   ├── podman/
+│   │   ├── containerfile_train     # FROM rocm/primus:v26.2
+│   │   ├── containerfile_serve     # FROM rocm/vllm-dev:rocm7.2.1
+│   │   └── digest.lock
+│   ├── compose/
+│   │   └── compose_dev.yaml
+│   └── k8s/
+│       └── train_job.yaml
+├── contracts/
+│   ├── foundry.toml
+│   ├── src/
+│   │   ├── mindxtrain_registry.sol  # immutable, no proxy, no admin
+│   │   └── x402_receiver.sol
+│   ├── script/
+│   └── test/
+├── examples/
+│   ├── demo_qwen3_8b_sft.yaml
+│   ├── demo_qwen3_6_27b_lora.yaml
+│   └── demo_grpo_gsm8k.yaml
+├── tests/
+│   ├── test_autotune.py
+│   ├── test_config.py
+│   ├── test_recipes.py
+│   └── test_receipt.py
+├── docs/
+│   ├── architecture.md
+│   ├── benchmarks.md
+│   └── hackathon_submission.md
+└── .github/workflows/
+    ├── ci_lint.yml
+    ├── ci_rocm_smoke.yml            # runs on self-hosted MI300X runner
+    └── publish_pypi.yml
+```
+
+## Critical code snippets
+
+**`pyproject.toml`** (excerpt):
+
+```toml
+[project]
+name = "mindxtrain"
+version = "0.1.0"
+description = "One-command Qwen3 fine-tuning on AMD MI300X"
+requires-python = ">=3.12"
+license = { text = "Apache-2.0" }
+authors = [{ name = "Gregory (codephreak)", email = "codephreak@pythai.net" }]
+dependencies = [
+  "typer>=0.12",
+  "pydantic>=2.7",
+  "pyyaml>=6.0",
+  "transformers>=4.46,<4.50",
+  "accelerate>=1.0",
+  "peft>=0.13",
+  "trl>=0.12",
+  "datasets>=3.0",
+  "optimum>=1.24",
+  "optimum-amd",
+  "amd-quark>=0.11.1",
+  "lm-eval>=0.4.5",
+  "optuna>=3.6",
+  "prometheus-client>=0.20",
+  "opentelemetry-api>=1.27",
+  "lighthouse-web3>=0.1",
+  "py-algorand-sdk>=2.6",
+  "ens>=0.5",
+  "datasketch>=1.6",        # MinHash dedupe
+  "numpy<2.0",
+]
+
+[project.optional-dependencies]
+axolotl   = ["axolotl @ git+https://github.com/axolotl-ai-cloud/axolotl@main"]
+unsloth   = ["unsloth"]
+torchtune = ["torchtune"]
+primus    = ["primus @ git+https://github.com/AMD-AGI/Primus@v26.2"]
+
+[project.scripts]
+mindxtrain = "mindxtrain.cli:app"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.ruff]
+line-length = 100
+target-version = "py312"
+```
+
+**`mindxtrain/cli.py`** (entry-point sketch):
+
+```python
+# SPDX-License-Identifier: Apache-2.0
+"""mindXtrain CLI — one-command Qwen3 fine-tuning on AMD MI300X."""
+from __future__ import annotations
+import typer
+from pathlib import Path
+from mindxtrain.config import load_config
+from mindxtrain.autotune.benchmark import run_benchmark
+from mindxtrain.autotune.plan import write_tuned_plan
+from mindxtrain.train.dispatch import dispatch_training
+from mindxtrain.eval.harness import run_eval
+from mindxtrain.quantize.quark_fp8 import quantize_fp8
+from mindxtrain.serve.vllm_rocm import serve_vllm
+from mindxtrain.publish.hf_hub import publish_to_hf
+from mindxtrain.publish.lighthouse import publish_to_lighthouse
+from mindxtrain.publish.mindx_register import register_with_mindx
+from mindxtrain.publish.agenticplace_list import list_on_agenticplace
+from mindxtrain.publish.bankon_ens import allocate_ens_subname
+from mindxtrain.receipt.manifest import emit_receipt
+
+app = typer.Typer(no_args_is_help=True, add_completion=False)
+
+@app.command()
+def init(project: str) -> None:
+    """Scaffold a new mindXtrain project (flat snake_case)."""
+    Path(project).mkdir(parents=True, exist_ok=False)
+    Path(f"{project}/config.yaml").write_text(_starter_yaml(project))
+    typer.echo(f"initialized {project}/")
+
+@app.command()
+def bench(
+    model: str = typer.Option(..., "--model"),
+    hardware: str = typer.Option("mi300x", "--hardware"),
+    gpus: int = typer.Option(1, "--gpus"),
+    seq_len: int = typer.Option(4096, "--seq-len"),
+    out: Path = typer.Option(Path("mindxtrain.tuned.yaml"), "--out"),
+) -> None:
+    """Run the 60-second MI300X autotune probe; emits an AOT plan."""
+    measurements = run_benchmark(model=model, hardware=hardware, gpus=gpus, seq_len=seq_len)
+    write_tuned_plan(measurements, out)
+    typer.echo(f"wrote tuned plan to {out}")
+
+@app.command()
+def train(config: Path = typer.Option(..., "-c", "--config")) -> None:
+    """Run the full pipeline (autotune → train → eval → quantize → publish)."""
+    cfg = load_config(config)
+    if cfg.autotune.enabled and not cfg.autotune.plan_path.exists():
+        bench(model=cfg.model.name, hardware=cfg.hardware.name,
+              gpus=cfg.hardware.gpus, seq_len=cfg.data.seq_len,
+              out=cfg.autotune.plan_path)
+    run_id = dispatch_training(cfg)
+    eval_report = run_eval(cfg, run_id)
+    if cfg.quantize.enabled:
+        quantize_fp8(cfg, run_id)
+    if cfg.publish.enabled:
+        hf_url = publish_to_hf(cfg, run_id, eval_report)
+        cid    = publish_to_lighthouse(cfg, run_id)
+        register_with_mindx(cfg, run_id, hf_url, cid)
+        list_on_agenticplace(cfg, run_id, hf_url)
+        allocate_ens_subname(cfg, run_id)
+    emit_receipt(cfg, run_id, eval_report)
+
+@app.command()
+def serve(
+    checkpoint: Path = typer.Option(..., "--checkpoint"),
+    backend: str = typer.Option("vllm-rocm", "--backend"),
+    port: int = typer.Option(8000, "--port"),
+) -> None:
+    """Serve a trained checkpoint on vLLM-ROCm or SGLang with correct parsers."""
+    if backend == "vllm-rocm":
+        serve_vllm(checkpoint, port=port)
+    else:
+        from mindxtrain.serve.sglang_rocm import serve_sglang
+        serve_sglang(checkpoint, port=port)
+
+if __name__ == "__main__":
+    app()
+```
+
+**`examples/demo_qwen3_8b_sft.yaml`** (the hackathon hero config, fits one MI300X):
+
+```yaml
+# SPDX-License-Identifier: Apache-2.0
+meta:
+  project: mindxtrain_demo
+  run_name: qwen3_8b_sft_demo
+  seed: 2048
+  license: apache-2.0
+
+hardware:
+  name: mi300x
+  gfx_arch: gfx942
+  gpus: 1
+  expected_hbm_gb: 192
+
+autotune:
+  enabled: true
+  plan_path: ./out/mindxtrain.tuned.yaml
+  budget_seconds: 60
+  policy: aot_only          # JIT autotune forbidden in production
+
+model:
+  name: Qwen/Qwen3-8B
+  attn_implementation: flash_attention_2  # CK backend by default; autotune may override
+  torch_dtype: bfloat16
+  trust_remote_code: false
+
+data:
+  source: hf
+  hf_id: HuggingFaceH4/ultrachat_200k
+  split: train_sft
+  seq_len: 4096
+  packing: true
+  dedupe:
+    minhash: { threshold: 0.85 }
+    semdedup: { threshold: 0.95, model: sentence-transformers/all-MiniLM-L6-v2 }
+  shard:
+    num_shards: 1
+
+train:
+  backend: axolotl          # autotune may flip to unsloth for single-GPU LoRA
+  method:
+    kind: lora
+    r: 16
+    alpha: 32
+    dropout: 0.0
+    target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]
+  optimizer:
+    name: adamw_torch_fused
+    lr: 1.0e-4
+    betas: [0.9, 0.95]
+    weight_decay: 0.1
+    grad_clip: 1.0
+  schedule:
+    type: cosine
+    warmup_ratio: 0.03
+    epochs: 3
+  batch:
+    per_device: 8
+    grad_accum: 4
+  precision: bf16
+  gradient_checkpointing: true
+  flash_attention:
+    backend: ck             # ck | triton | aiter — autotune picks
+  fsdp: { enabled: false }  # single GPU
+  env:
+    HSA_NO_SCRATCH_RECLAIM: "1"
+    NVTE_CK_USES_BWD_V3: "1"
+    NVTE_CK_IS_V3_ATOMIC_FP32: "1"
+    PRIMUS_TURBO_ATTN_V3_ATOMIC_FP32: "1"
+    NCCL_MIN_NCHANNELS: "112"
+    HIP_FORCE_DEV_KERNARG: "1"
+    PYTORCH_ROCM_ARCH: "gfx942"
+
+eval:
+  harness:
+    tasks: [mmlu, gsm8k, ifeval, humaneval]
+    fewshot: 5
+  regression:
+    baseline: Qwen/Qwen3-8B
+    threshold_pct: -1.0     # fail if any task drops more than 1 pct
+
+quantize:
+  enabled: true
+  scheme: quark_fp8         # quark_fp8 | quark_mxfp4 | gptq_rocm
+  ptpc: true                # PTPC FP8 GEMM (15-30% faster than BlockScale on MI300X)
+
+serve:
+  backend: vllm-rocm
+  reasoning_parser: deepseek_r1   # qwen3 for 3.5/3.6
+  tool_call_parser: hermes        # qwen3_coder for Coder family
+  tensor_parallel: 1
+
+publish:
+  enabled: true
+  hf:
+    repo: pythai/qwen3-8b-mindxtrain-demo
+    private: false
+  lighthouse:
+    api_key_env: LIGHTHOUSE_API_KEY
+  mindx:
+    api_url: https://mindx.pythai.net/v1/agents
+    register_as_capability: true
+  agenticplace:
+    api_url: https://agenticplace.pythai.net/v1/listings
+    chain_map_url: https://agenticplace.pythai.net/allchain.html
+  bankon:
+    ens_parent: bankon.eth
+    subname: qwen3-8b-mindxtrain-demo
+  billing:
+    x402:
+      network: algorand
+      asset: USDC
+      receiver_via: parsec_wallet
+      price_per_1k_tokens: 0.0002
+
+receipt:
+  output: ./out/receipt.json
+  include:
+    [rocm_version, gfx_arch, container_digest, all_git_shas,
+     yaml_hash, dataset_cids, eval_report, energy_kwh]
+```
+
+**Sample Solidity registry stub (Foundry, immutable, no proxy, no EOA admin)**:
+
+```solidity
+// SPDX-License-Identifier: Apache-2.0
+pragma solidity ^0.8.26;
+
+contract MindXTrainRegistry {
+    struct Receipt {
+        bytes32 yamlHash;
+        bytes32 datasetCidHash;
+        bytes32 checkpointCidHash;
+        bytes32 evalReportHash;
+        address publisher;
+        uint64  timestamp;
+    }
+
+    mapping(bytes32 => Receipt) private _receipts;
+    event ReceiptAnchored(bytes32 indexed runId, address indexed publisher, bytes32 yamlHash);
+
+    function anchor(
+        bytes32 runId,
+        bytes32 yamlHash,
+        bytes32 datasetCidHash,
+        bytes32 checkpointCidHash,
+        bytes32 evalReportHash
+    ) external {
+        require(_receipts[runId].timestamp == 0, "exists");
+        _receipts[runId] = Receipt({
+            yamlHash: yamlHash,
+            datasetCidHash: datasetCidHash,
+            checkpointCidHash: checkpointCidHash,
+            evalReportHash: evalReportHash,
+            publisher: msg.sender,
+            timestamp: uint64(block.timestamp)
+        });
+        emit ReceiptAnchored(runId, msg.sender, yamlHash);
+    }
+
+    function get(bytes32 runId) external view returns (Receipt memory) {
+        return _receipts[runId];
+    }
+}
+```
+
+No constructor admin, no `Ownable`, no upgradeable proxy, no pause function, no setter — write-once anchoring, EOA-key-free administration. DAIO blockchain deployment is the remaining piece per the user's standard preferences; mainnet Foundry deploy targets the canonical chain ID resolved through `agenticplace.pythai.net/allchain.html`.
+
+## Reconciliation with prior `mindXtrain.md` and `mindXtrain2.md`
+
+Because the two prior design files are not in research context, this document is written as a forward-compatible **continuation**, not a replacement. Three places need explicit reconciliation when the prior files are read alongside this one. **First**, the AMD-track framing pulls the recommended default model away from any GPU-agnostic choice toward Qwen3.6-27B / Qwen3.6-35B-A3B / Qwen3-8B specifically, because Qwen integration is a stackable hackathon prize and Qwen3 has confirmed Day-0 ROCm support; if mindXtrain.md/mindXtrain2.md committed to a different default base model, that decision should be revisited for the hackathon submission only and reverted afterwards if needed. **Second**, the AOT-only autotune discipline carried forward from the user's mindX AutoTune work conflicts with any framework default that turns on JIT autotune (Triton autotune in vLLM cold-start, `torch.compile(mode='max-autotune')` Inductor JIT, MIOpen find-mode); the mindXtrain config schema must explicitly disable JIT autotune in production runs and force AOT compilation paths (AOTriton, hipBLASLt offline tune cache, MIOpen `.kdb` pre-warming). **Third**, the BANKON ENS allocation and x402-Algorand billing flow is named here as a first-class integration; if the prior files described mindXtrain as standalone, this needs to be elevated from optional to mandatory in the publish step, and the cypherpunk2048 immutability rule (no upgradeable proxies, no EOA admin) must propagate into the on-chain registry contract above.
+
+## Hackathon timeline (today is May 5 2026)
+
+The build window has roughly five days of active engineering left. **May 5 (today)**: register on lablab.ai, register the AMD AI Developer Program for the $100 cloud credits, provision an MI300X via TensorWave bare-metal or AMD Developer Cloud, snapshot the `rocm/primus:v26.2` digest, scaffold the `mindxtrain/` repo with the directory tree above, ship the Pydantic config schema and the Typer CLI, post a Build-in-Public X teaser tagging @lablab @AIatAMD with a screenshot of `mindxtrain init`. **May 6**: implement the autotune layer — attention probe, GEMM probe, RCCL probe, plan emitter — and run the first end-to-end Qwen3-8B SFT-LoRA on a single MI300X with the demo YAML; capture tok/s and MFU baseline numbers. **May 7**: implement the dataset pipeline (MinHash, SemDeDup, packing, sharding), the eval harness wrapper, and the Quark FP8 PTPC quantization path; ship a second X post showing the autotune-driven config diff with a benchmark vs untuned baseline. **May 8**: implement the publish layer (HF, Lighthouse, mindX register, AgenticPlace listing, BANKON ENS subname), the x402 billing stub, the receipt manifest, and deploy the public HF Space inside `lablab-ai-amd-developer-hackathon`; record the demo video and write the final pitch deck. **May 9**: travel to SF or stream to the on-site session, finalize the lablab submission form by 14:00 local Saturday, drive social engagement on the HF Space for the most-likes prize. **May 10**: live demo on stage, awards, post-mortem. Submit the written ROCm developer-experience feedback note immediately after submission to lock in Build-in-Public eligibility.
+
+## Closing synthesis
+
+mindXtrain wins this hackathon by being the only entry that operationalizes the *entire* AMD training stack — ROCm 7.2.1, AOTriton, AITER, Composable Kernel, hipBLASLt, RCCL, Optimum-AMD, Quark, Primus-Turbo, vLLM-ROCm, SGLang — behind a single CLI, with a defensible **AOT autotune** that nobody else in the ecosystem ships, against the **latest open Qwen3.6 checkpoints** (which the user correctly remembered as real and which most public summaries lag), with cost numbers that make MI300X look obviously cheaper than H100 for the workloads in scope. The cypherpunk2048 discipline — Apache 2.0, flat snake_case, Podman, immutable contracts, no proxies, no EOA admin — is preserved end-to-end, and the integration plumbing into mindX, AgenticPlace, BANKON-ENS and x402-Algorand is wired without locking in a proprietary dependency. The remaining engineering is entirely tractable in the five-day window, and every external dependency named in this blueprint has either a verified ROCm-first-class status or a documented community workaround. Ship it.
+
+---
+
+### Citations
+
+**Hackathon**: lablab.ai/ai-hackathons/amd-developer · amd.com/en/developer/resources/technical-articles/2026/build-across-the-ai-stack--join-the-amd-x-lablab-ai-hackathon-.html · luma.com/afz0aeq8 · huggingface.co/lablab-ai-amd-developer-hackathon · lablab.ai/ai-articles/from-zero-to-ai-builder-amd-developer-program · lablab.ai/ai-tutorials/amd-developer-cloud-host-llm-vllm
+
+**Frameworks**: github.com/axolotl-ai-cloud/axolotl · github.com/AI-DarwinLabs/axolotl · github.com/hiyouga/LLaMA-Factory · github.com/unslothai/unsloth · github.com/pytorch/torchtune · github.com/huggingface/trl · github.com/huggingface/peft · github.com/huggingface/accelerate · github.com/huggingface/optimum-amd · github.com/microsoft/DeepSpeed · github.com/AMD-AGI/Primus · github.com/AMD-AGI/Primus-Turbo · github.com/vllm-project/vllm · github.com/sgl-project/sglang
+
+**AMD stack**: rocm.docs.amd.com/en/latest/about/release-notes.html · rocm.docs.amd.com/en/latest/compatibility/compatibility-matrix.html · rocm.docs.amd.com/projects/install-on-linux/en/latest/install/3rd-party/pytorch-install.html · github.com/ROCm/bitsandbytes (rocm_enabled_multi_backend) · github.com/Dao-AILab/flash-attention · github.com/ROCm/aotriton · github.com/ROCm/aiter · github.com/ROCm/composable_kernel · github.com/amd/Quark · quark.docs.amd.com · rocm.blogs.amd.com/software-tools-optimization/mi300x-rccl-xgmi · rocm.blogs.amd.com/software-tools-optimization/vllm-omni · rocm.blogs.amd.com/software-tools-optimization/llm-grpo-rocm · rocm.blogs.amd.com/artificial-intelligence/qwen3-day0-amd · rocm.blogs.amd.com/artificial-intelligence/torchtune · www.amd.com/en/products/accelerators/instinct/mi300/mi300x.html · www.amd.com/en/products/accelerators/instinct/mi350.html · www.amd.com/en/developer/resources/technical-articles/2026/day-0-support-for-qwen-3-5-on-amd-instinct-gpus.html · huggingface.co/blog/huggingface-and-optimum-amd · huggingface.co/blog/microsoft-collaboration · huggingface.co/amd · huggingface.co/docs/optimum/en/amd/index · arxiv.org/pdf/2510.27583 · newsletter.semianalysis.com/p/mi300x-vs-h100-vs-h200-benchmark-part-1-training
+
+**Qwen3**: arxiv.org/abs/2505.09388 · arxiv.org/abs/2509.17765 · arxiv.org/abs/2511.21631 · qwenlm.github.io/blog/qwen3 · qwen.ai/blog?id=qwen3-next · qwen.ai/blog?id=qwen3.5 · qwen.ai/blog?id=qwen3.6-27b · qwen.ai/blog?id=qwen3.6-35b-a3b · github.com/QwenLM/Qwen3 · github.com/QwenLM/Qwen3-Coder · github.com/QwenLM/Qwen3-VL · github.com/QwenLM/Qwen3-Omni · github.com/QwenLM/Qwen3.6 · huggingface.co/Qwen · huggingface.co/Qwen/Qwen3-8B · huggingface.co/Qwen/Qwen3-32B · huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507 · huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct · huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct · huggingface.co/Qwen/Qwen3.6-27B · huggingface.co/Qwen/Qwen3.6-35B-A3B · qwen.readthedocs.io/en/latest/getting_started/quickstart.html · qwen.readthedocs.io/en/latest/getting_started/concepts.html · www.lmsys.org/blog/2026-02-11-Qwen-latency · docs.unsloth.ai/models/qwen3-how-to-run-and-fine-tune · unsloth.ai/docs/models/qwen3.6

docs/blueprints/mindXtrain_ Production Blueprint for the AMD and lablab.ai Hackathon.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ab7d4053c77df1e291a7f48dace094e39a6d09a79b6176050ada5f36d97c5db
+size 875047

docs/cli.md

@@ -0,0 +1,208 @@
+# CLI reference
+
+The `mindxtrain` Typer app: 9 verbs (8 top-level + a `dataset` subgroup).
+Every verb that consumes a YAML config validates it against the
+[10-section schema](yaml_schema.md) before doing anything else.
+
+```
+mindxtrain [--version] <verb> [options]
+```
+
+All verbs dispatch into real Python in the canonical `mindxtrain.*` modules.
+Verbs that require optional dependencies surface a clean
+`run `uv sync --extra <group>`` hint and exit `3`.
+
+## Global options
+
+| Flag         | Purpose                              |
+|--------------|--------------------------------------|
+| `--version`  | Print the version and exit.          |
+| `--help`     | Show help for the top-level command. |
+
+## `init` — scaffold a YAML
+
+Render a built-in recipe to disk.
+
+```
+mindxtrain init [--template <name>] [--out <path>] [--list]
+```
+
+| Option                | Default              | Description                                      |
+|-----------------------|----------------------|--------------------------------------------------|
+| `--template`, `-t`    | `qwen3_8b_sft_lora`  | recipe name; see `--list`                        |
+| `--out`, `-o`         | `run.yaml`           | output path                                      |
+| `--list`              | _flag_               | print every built-in recipe and exit             |
+
+Available recipes (12 total):
+
+```
+instella_3b_lora        qwen3_30b_a3b_lora      qwen3_32b_dpo
+qwen3_32b_full_fsdp     qwen3_32b_grpo          qwen3_32b_orpo
+qwen3_6_27b_lora        qwen3_6_35b_a3b_lora    qwen3_8b_cpt
+qwen3_8b_sft_full       qwen3_8b_sft_lora       qwen3_vl_8b_sft
+```
+
+```bash
+$ uv run mindxtrain init --template qwen3_8b_sft_lora --out run.yaml
+wrote run.yaml (2785 bytes, recipe='qwen3_8b_sft_lora')
+```
+
+## `bench` — run the 60-second AOT autotune probe
+
+The differentiator. See [autotune.md](autotune.md) for probe taxonomy.
+
+```
+mindxtrain bench [--gpu N] [--out <path>] [--dry-run]
+```
+
+| Option       | Default                | Description                                                         |
+|--------------|------------------------|---------------------------------------------------------------------|
+| `--gpu`      | `0`                    | HIP/ROCm device index                                               |
+| `--out`, `-o`| `autotune_plan.json`   | output path                                                         |
+| `--dry-run`  | _flag_                 | skip GPU probes; emit a synthetic reference plan (CPU-safe)         |
+
+`--dry-run` is the CPU-only path used in tests and CI. A real
+`mindxtrain bench --gpu 0` requires `torch` (`--extra ml`) and an MI300X
+with ROCm 7.2.1; if torch is unavailable, the attention probe gracefully
+falls back to the canonical `ck` default.
+
+## `train` — dispatch a training run
+
+```
+mindxtrain train <config.yaml> [--plan <plan.json>] [--out <run-dir>]
+```
+
+| Option       | Default                | Description                                                  |
+|--------------|------------------------|--------------------------------------------------------------|
+| `--plan`     | (uses dry-run plan)    | autotune plan JSON from `mindxtrain bench`                   |
+| `--out`, `-o`| `./out/runs`           | output root for `<run_id>/` directory                        |
+
+Loads the YAML, dispatches to `train.backend` (`axolotl`, `unsloth`,
+`torchtune`, `primus`). The Axolotl path subprocess-wraps
+`accelerate launch -m axolotl.cli.train`. Plan-derived env vars
+(`PYTORCH_ROCM_ARCH=gfx942`, `HSA_NO_SCRATCH_RECLAIM=1`, etc.) are injected
+before launch.
+
+Requires `--extra ml` plus the chosen backend on `PATH`.
+
+Exits `3` with a clean install hint if `accelerate` (or the backend) is missing.
+
+## `dataset prep` — run the dataset pipeline
+
+```
+mindxtrain dataset prep <config.yaml> [--out <dir>]
+```
+
+Streams the HF dataset (`datasets`), runs heuristic + optional MinHash/SemDeDup
+filters, tokenizes (`AutoTokenizer`), packs to `data.seq_len`, emits sharded
+`.tar` files. Pin the resulting tars via
+`mindxtrain.storage.lighthouse` or `mindxtrain.storage.ipfs`.
+
+Requires `--extra ml` (datasets, transformers).
+
+## `eval` — run lm-evaluation-harness
+
+```
+mindxtrain eval <config.yaml> [--checkpoint <path>]
+```
+
+| Option       | Default                                           | Description                |
+|--------------|---------------------------------------------------|----------------------------|
+| `--checkpoint`, `-c` | `./out/runs/<run_name>/checkpoint`        | path to the checkpoint dir |
+
+Subprocess-wraps `lm_eval --model hf --tasks <comma-sep>`. Tasks come from
+`cfg.eval.harness.tasks`. Output JSON written under
+`<checkpoint>/eval/lm_eval.json`. Summary printed via
+`mindxtrain.eval.harness.parse_summary`.
+
+Requires `--extra eval`.
+
+## `quantize` — Quark FP8 / MXFP4
+
+```
+mindxtrain quantize <config.yaml> [--checkpoint <path>]
+```
+
+Wraps `python -m amd_quark.quantize` with `--scheme fp8_e4m3` (default) or
+`--scheme mxfp4` (CDNA 4 / MI350X+). Output is a `quantized/` directory next
+to the checkpoint, vLLM-loadable.
+
+Requires the `amd-quark` package — typically only available inside the
+`rocm/primus:v26.2` container or per
+[Quark docs](https://quark.docs.amd.com/).
+
+## `serve` — print the vLLM-ROCm launch command
+
+```
+mindxtrain serve <config.yaml> [--checkpoint <path>]
+```
+
+Builds the `vllm serve` argv from `cfg.serve` and prints it. We deliberately
+don't `exec` — the user pipes it into their own orchestrator (or
+`ops/compose/compose_dev.yaml`).
+
+The chat-template parsers map per `serve.reasoning_parser` (`qwen3` for Qwen3,
+`deepseek_r1` for DeepSeek-style) and `serve.tool_call_parser` (`hermes`,
+`qwen3_coder`).
+
+## `publish` — push to HF + Lighthouse + register
+
+```
+mindxtrain publish <config.yaml> --manifest <manifest.json> [--skip-hf] [--skip-pin]
+```
+
+1. `mindxtrain.storage.hf_hub.publish_to_hf` — uploads the checkpoint dir to
+   HuggingFace Hub (uses `HF_TOKEN`). `--skip-hf` to bypass.
+2. `mindxtrain.storage.lighthouse.publish_to_lighthouse` — pins to
+   Lighthouse via direct httpx POST (uses `LIGHTHOUSE_API_KEY`). Falls back
+   to a stub `cid://stub-...` derived from the checkpoint's BLAKE3 if the
+   key is unset. `--skip-pin` to bypass entirely.
+3. `mindxtrain.deploy.api_client.register_with_mindx` — POSTs the run-id /
+   HF URL / CID to `MINDXTRAIN_API_BASE_URL/v1/agents`. Skipped gracefully
+   if the endpoint isn't reachable.
+4. The manifest JSON file is updated in-place with the resulting `hf_repo_id`
+   and `lighthouse_cid` fields.
+
+## `receipt` — verify a provenance manifest
+
+```
+mindxtrain receipt <manifest.json> [--config <run.yaml>]
+```
+
+Loads the manifest and prints the run-id + BLAKE3 fields. With `--config`,
+also re-hashes the on-disk artifacts (`config_yaml`, `dataset`, `checkpoint`,
+`eval_json`) and emits a per-field pass/fail dict — exits `0` if every hash
+verifies, `2` if any drift is detected.
+
+```bash
+$ uv run mindxtrain receipt out/runs/<run_id>/manifest.json --config run.yaml
+{
+  "config_yaml": true,
+  "dataset": true,
+  "checkpoint": true,
+  "eval_json": true
+}
+```
+
+## Exit-code summary
+
+| Code | Meaning                                                         |
+|------|-----------------------------------------------------------------|
+| 0    | Success.                                                        |
+| 1    | Bad input — missing file, hash mismatch, schema error.          |
+| 2    | Verify failed — at least one BLAKE3 field doesn't match disk.   |
+| 3    | Optional dep missing — install with `uv sync --extra <group>`.  |
+
+## Where the verbs live
+
+| Verb              | Module                                                                                |
+|-------------------|---------------------------------------------------------------------------------------|
+| `init`            | `mindxtrain.cli.main.init` + `mindxtrain.config.loader.render_recipe`                 |
+| `bench`           | `mindxtrain.cli.main.bench` + `mindxtrain.autotune.benchmark.run_autotune`            |
+| `train`           | `mindxtrain.cli.main.train` + `mindxtrain.train.dispatch.dispatch_training`           |
+| `dataset prep`    | `mindxtrain.cli.main.dataset_prep` + `mindxtrain.data.{curate,filter,tokenize,pack}`  |
+| `eval`            | `mindxtrain.cli.main.eval_` + `mindxtrain.eval.harness.run_lm_eval`                   |
+| `quantize`        | `mindxtrain.cli.main.quantize` + `mindxtrain.deploy.quark.quark_fp8`                  |
+| `serve`           | `mindxtrain.cli.main.serve` + `mindxtrain.deploy.vllm_launcher.build_vllm_command`    |
+| `publish`         | `mindxtrain.cli.main.publish` + `mindxtrain.storage.{hf_hub,lighthouse}` + `mindxtrain.deploy.api_client` |
+| `receipt`         | `mindxtrain.cli.main.receipt` + `mindxtrain.provenance.verify.verify_receipt`         |

docs/coach.md

@@ -0,0 +1,260 @@
+# mindxtrain Coach (UI)
+
+A single-page web UI that walks judges and new contributors through the mindxtrain pipeline without needing a GPU. Bundled inside the mindxtrain.operator FastAPI app at `/coach/`.
+
+## Why it exists
+
+Hackathon judges have ~3 minutes per submission. The Coach lets them poke at the differentiator (the 60-second AOT autotune) and the cost story (4× cheaper than H100) interactively, in a browser, without setting up ROCm.
+
+## Boot
+
+```bash
+uv run uvicorn mindxtrain.operator.app:app --host 0.0.0.0 --port 8080
+```
+
+Open http://localhost:8080 — the root path redirects to `/coach/`.
+
+The Coach works **without** a backend GPU:
+- The autotune endpoint runs `run_autotune(dry_run=True)` and emits the reference plan.
+- The compile endpoint produces a real Axolotl YAML against the dry-run plan.
+- The cost calculator is pure arithmetic.
+
+The chat panel stays disabled until `MINDXTRAIN_BACKEND=vllm` is set and a vLLM-ROCm server is reachable.
+
+## Layout
+
+```
+mindxtrain/operator/coach/
+├── __init__.py            # exports the FastAPI router
+├── api.py                 # routes (recipes / bench / compile / cost / health /
+│                          #   runs / metrics / receipt / sea-decision / mei / diagnostics)
+├── run_metrics.py         # 1 Hz system-metrics sampler (psutil + /proc)
+├── chronos_client.py      # mindX promised-time client
+└── static/
+    ├── index.html         # multi-card SPA shell (preflight → … → train → receipt → chat)
+    ├── style.css          # minimal dark-friendly CSS, AMD orange accent
+    └── coach.js           # vanilla JS state machine, no framework
+```
+
+The Coach mounts under `/coach/`; static assets are at `/coach/static/*`. The UI
+has grown well past the original five-step demo: it now covers preflight, hardware
+detection, dream-corpus stats, recipe pick, autotune, compile, **live training with
+diagnostic feedback**, the **verifiable receipt**, MEI scoring, cost, deploy, and chat.
+
+## Routes
+
+| Method | Path                                | Body / Query              | Returns                                    |
+|--------|-------------------------------------|---------------------------|--------------------------------------------|
+| GET    | `/`                                 | —                         | 307 redirect to `/coach/`                  |
+| GET    | `/coach/`                           | —                         | `index.html`                               |
+| GET    | `/coach/static/{path}`              | —                         | static files                               |
+| GET    | `/coach/api/recipes`                | —                         | `list[RecipeSummary]` (12 items)           |
+| GET    | `/coach/api/recipes/{name}`         | —                         | `{ name, yaml, summary }`                  |
+| POST   | `/coach/api/bench`                  | (none)                    | `AutotunePlan` (dry-run reference)         |
+| POST   | `/coach/api/compile`                | `{recipe, plan?}`         | `{recipe, config_summary, plan, axolotl_yaml, overrides}` |
+| POST   | `/coach/api/cost`                   | `{gpus, hours, safety_margin}` | `{mi300x, h100, h200, speedup_vs_h100_x}`  |
+| GET    | `/coach/api/health`                 | —                         | `{coach_version, chat_backend_ready, recipes_available}` |
+| POST   | `/coach/api/runs/launch`            | `{recipe, plan?, out_dir?}` | `Run` snapshot (spawns training)         |
+| GET    | `/coach/api/runs/{id}/events`       | —                         | SSE stream (`status`/`step`/`eval`/`log`/`metrics`/`energy`) |
+| GET    | `/coach/api/runs/{id}/metrics`      | `?since=`                 | system-metrics backfill for the sparklines |
+| GET    | `/coach/api/receipt/{run_id}`       | —                         | `ReceiptView` — re-verified BLAKE3 hashes + `verified` |
+| GET    | `/coach/api/sea-decision`           | —                         | mindX SEA autonomous-training gate state   |
+| GET    | `/coach/api/mei/score/{run_id}`     | —                         | `MEIScoreView` (mindX Efficiency Index)    |
+| GET    | `/coach/api/diagnostics/live`       | —                         | host load / RAM% / disk% / operator RSS    |
+
+The full schema is rendered at `/docs` (Swagger).
+
+## Live training diagnostics
+
+The **Train (live)** card is the accurate, real-time depiction of a run. Events
+arrive over Server-Sent Events (`/coach/api/runs/{id}/events`) — `step`, `eval`,
+`log`, and 1 Hz `metrics` — and drive these surfaces:
+
+- **Session headline** — status badge, wall-clock + CPU-time elapsed, throttle%,
+  last loss, freshest eval. The at-a-glance "is it healthy" line.
+- **Phase + progress** — friendly phase narration ("Loading base model…",
+  "Training…", "Saving checkpoint…") plus a progress bar with `step N / total · ETA`,
+  driven by `StepEvent.total_steps`.
+- **Loss curve** (Chart.js) — dual-axis loss (orange) + `mean_token_accuracy`
+  (green, NaN-gapped where a backend omits it). The primary "is it learning" signal.
+
+Because a real MI300X run logs **thousands of steps**, the heavy detail is kept
+accurate but compressed behind accordions, with the truncation always shown — never
+silent:
+
+- **Loss curve** keeps a rolling window of the last `MAX_CHART_POINTS` (1500) points;
+  once it rolls, a `showing last 1500 of N steps` note appears under the chart.
+- **Per-step metrics** (step, loss, acc, entropy, lr, grad_norm) live in a collapsed
+  `<details>` accordion; the DOM table caps at 50 rows but the summary reports the
+  true total — `per-step metrics (N steps · last 50 shown)`.
+- **train.log (live tail)** is a `<details>` accordion that auto-folds older lines
+  and shows a running `(N lines)` count, capping the DOM at `MAX_LOG_LINES` (2000)
+  and labelling `· oldest dropped` once it does.
+- **System metrics** — five d3 sparklines (host cpu%/ram%/load, trainer rss MB,
+  trainer cpu-s/s) sampled at 1 Hz, in their own `<details>` (open by default).
+
+This keeps the page legible on a laptop while the underlying data stays faithful.
+
+## Verifiable receipt card
+
+When a run finishes, the operator emits `manifest.json` (BLAKE3 of the config
+snapshot, checkpoint, and the frozen `AutotunePlan`) into the run directory. The
+**Verifiable receipt** card fetches `/coach/api/receipt/{run_id}`, which re-hashes
+the on-disk artifacts and returns a `verified` flag plus the per-field checks. A
+`verified ✓` badge and the truncated hashes render in the card; the same check runs
+from a shell via `mindxtrain receipt out/runs/<run>/manifest.json --config <recipe>.yaml`.
+Binding the AutotunePlan hash to the checkpoint is the AOT-as-verification primitive —
+it proves which compiled backend/heuristic/RCCL config produced the weights.
+
+## Create script + imprint (actor / persona / script)
+
+mindXtrain (and Coach) **train models**. The model is an **actor**; an actor has a
+**persona** (identity / voice) and a **script** (the training examples — the
+"impression"). The **Create script** card authors a small script in the browser and
+saves it as `source: local` JSONL the recipes ingest.
+
+- **`POST /coach/api/datasets`** — `{name, persona_name, system_prompt, voice_examples,
+  exchanges:[{user,assistant}], seed_voice}` → writes
+  `out/datasets/<name>/script.jsonl` (override the root with `MINDXTRAIN_DATASETS_DIR`).
+  `GET /coach/api/datasets` lists them; `GET /coach/api/datasets/{name}` previews.
+- **`GET /coach/api/persona`** — pre-fills the form from `MINDXTRAIN_PERSONA_PATH`
+  (clean-room: recognised fields only, never copies mindX bytes).
+- Point the **`mindx_persona_imprint_local`** recipe's `data.path` at the saved script
+  and train the tiny actor (`trl_local`, CPU or local GPU).
+
+**Imprint = recall, before vs after.** Pose the script's own user-turns back to the
+actor and compare the base model (before) with the trained adapter (after) against the
+script's assistant voice:
+
+```bash
+mindxtrain imprint mindxtrain/train/recipes/mindx_persona_imprint_local.yaml
+```
+
+prints an `ImprintReport` (`before_voice`, `after_voice`, `imprint_delta`, `shift`,
+`imprinted`); exit 4 if no imprint took. `POST /coach/api/imprint/score` scores supplied
+utterances without blocking the event loop on inference. `mindxtrain imprint
+--trigger-dream` hands the imprinted actor to mindX's `machine.dream` 8-hour cycle (via
+`MINDXTRAIN_API_BASE_URL` `/v1/dream/ingest`, else a `data/incoming/` inbox drop under
+`MINDXTRAIN_MINDX_HOME`) — clean-room, an artifact pointer, never mindX code.
+
+## Create script — personas + skills
+
+The **Create script** card authors a `source: local` JSONL from a persona and toggleable
+skills:
+
+- **Built-in personas** (`GET /coach/api/personas`) — `codephreak`, `assistant`, `mentor`
+  (`mindxtrain.data.personas.BUILTIN_PERSONAS`). Pick one, or use the custom fields.
+- **Skills** — toggle **Software Engineer / Platform Architect / Bash / Solidity** to mix
+  each skill's in-domain exchanges into the script (`mindxtrain.data.personas.SKILLS`,
+  `compose(persona, skills)`). A skill is a system-prompt addendum + representative turns.
+- `POST /coach/api/datasets` composes persona + skills + your exchanges and returns the row
+  count plus **training params auto-derived from the dataset size**
+  (`derive_training_params` — small scripts overfit to imprint: more epochs, grad_accum 1).
+
+## Build an Ollama Modelfile (separate window)
+
+The **Build Modelfile…** button (in the train card's push-to-ollama row) opens a standalone
+builder at `/coach/modelfile` (a separate browser window), pre-filled for the current run:
+
+- Every instruction is a toggle: `FROM` (required), `SYSTEM`, `TEMPLATE`, `ADAPTER`,
+  `LICENSE`, `REQUIRES`, plus `MESSAGE` examples and `stop` sequences.
+- Every `PARAMETER` (`num_ctx`, `temperature`, `top_k`, `top_p`, `min_p`, `repeat_penalty`,
+  `mirostat`, `seed`, … — the full catalogue from `GET /coach/api/modelfile/params`) is a
+  toggle + input, rendered dynamically with defaults and ranges.
+- `POST /coach/api/modelfile/build` renders the `Modelfile` text;
+  `POST /coach/api/modelfile/create` runs `ollama create <tag>`. Core logic:
+  `mindxtrain.deploy.modelfile` (`ModelfileSpec`, `render_modelfile`, `create_model`).
+
+## The core storyboard
+
+The original CPU-only demo path, top-to-bottom (the cards above and below it —
+preflight, hardware, dream-corpus, live training, receipt, MEI, deploy — flank it):
+
+1. **Pick a recipe** — clickable grid of all built-in recipes; the selected one's YAML expands inline.
+2. **Run the autotune probe** — single button; shows the `AutotunePlan` JSON plus a six-chip summary (`attention=ck`, `gemm=hipblaslt_default`, `rccl=1gpu_noop`, …).
+3. **Compile to Axolotl YAML** — translates `(recipe, plan)` into the trainer-side YAML, surfaces the plan-driven overrides as chips above the YAML.
+4. **Train (live)** — spawns the run and streams the diagnostic feedback described in [Live training diagnostics](#live-training-diagnostics); on a CPU box the `trl_cpu` lane trains a small model in-process so the whole loop is demoable without a GPU. The `trl_local` lane is the device-aware variant — it uses a local consumer GPU (CUDA or ROCm Radeon) when present and falls back to CPU otherwise, so the same recipe runs on a laptop or a gaming GPU. `recommend_lane` sends an Instinct/MI300X card to `axolotl_amd` and any other local GPU to `trl_local`.
+5. **Verifiable receipt** — the `verified ✓` badge + bound hashes appear the moment the run completes.
+6. **Cost vs H100** — sliders for GPUs and hours; emits a three-row comparison table (MI300X / H100 / H200) with a headline like "MI300X is 5.4× cheaper than the H100 baseline".
+7. **Try the model** — chat panel that proxies to `/v1/chat/completions`. Stays disabled and explains why until the backend reports ready; a **Check now** button re-probes on demand.
+
+## Demo storyboard
+
+```
+0:00–0:30  open localhost:8080, point at the three-stage diagram in the header
+0:30–1:00  click qwen3_8b_sft_lora; show the YAML preview
+1:00–2:00  click "Run autotune (dry-run)"; show the plan JSON streaming in
+           and the six-chip summary populating
+2:00–3:00  click "Compile"; show the Axolotl YAML diff (the autotune
+           plan's attention_backend appears as flash_attn_backend=ck)
+3:00–4:00  drag the cost slider to 1 GPU × 1.5 hours; show the
+           "5× cheaper than H100" headline
+4:00–5:00  the chat panel; show that it's gracefully disabled because
+           the backend isn't booted, then close
+```
+
+Every Coach interaction is screen-recordable on a CPU-only laptop. The MI300X work happens behind the scenes for the actual training run; the Coach surfaces the *outcome* judges care about.
+
+## Dependencies
+
+- FastAPI — already a dep of mindxtrain.operator.
+- `mindxtrain` — workspace dep added to `pyproject.toml` so the Coach can call `mindxtrain.config.loader.list_recipes()`, `mindxtrain.autotune.benchmark.run_autotune()`, and `mindxtrain.train.compile_axolotl_yaml()`.
+- `pyyaml` — added for the recipe→summary path.
+
+No JavaScript framework, no build step, no node_modules.
+
+## Tests
+
+`tests/test_coach_api.py` covers every endpoint via FastAPI's `TestClient`:
+
+- root redirects to `/coach/`
+- index serves HTML with the right `<title>`
+- static files serve (CSS + JS)
+- recipes list returns 12 items
+- recipe detail returns YAML + summary
+- 404 on unknown recipe
+- bench returns a valid `AutotunePlan`
+- compile returns Axolotl YAML + overrides; 404 on unknown recipe
+- cost returns three breakdowns; 422 on invalid input
+- health endpoint reports `recipes_available=12`
+- `/health` mentions `coach_url=/coach/`
+- the train card exposes the diagnostic accordions (`metrics-table-wrap`,
+  `metrics-table-count`, `train-log-count`, `chart-window-note`) and coach.js wires
+  the rolling-window cap + counters (`MAX_CHART_POINTS`, `_updateMetricsTableCount`,
+  `_updateLogCount`)
+- the receipt card + loader are present (`step-receipt`, `loadReceiptForRun`)
+
+The live-training + receipt round-trip is covered in `tests/test_coach_receipt_api.py`
+(canned spawn → `/coach/api/receipt/{id}` returns `verified=True`).
+
+Run with `uv run pytest tests/test_coach_api.py -v`.
+
+## Customizing for the demo
+
+Tweak the cost-comparison constants in `mindxtrain/operator/coach/api.py`:
+
+```python
+H100_USDC_PER_HOUR = 4.00
+H200_USDC_PER_HOUR = 6.00
+```
+
+The MI300X rate is sourced from `mindxtrain.budget.pricing.MI300X_USDC_PER_HOUR` ($1.99/hr, AMD Developer Cloud list price).
+
+## Streaming chat + ollama controls (Try the model)
+
+The **Try the model** card chats with a local model and **streams the response
+token-by-token** — the [AI SDK](<Vercel AI SDK 6_ A Framework-Agnostic Deep Dive (June 2026).md>)
+text-stream pattern, implemented in vanilla JS (no build step): `coach.js` consumes a
+`text/event-stream` whose `data:` lines are JSON token deltas, ending with `data: [DONE]`.
+
+- **`POST /coach/api/chat/stream`** — `{model, messages, max_tokens?}` → SSE token stream.
+  Relays `backend.stream_chat()` (the OpenAI-compatible streaming the ollama/vLLM backends
+  already speak). Backend errors are surfaced in-stream (`event: error`), never as a mid-stream 500.
+- **Model picker** — populated from `GET /coach/api/models` (local models sorted ahead of
+  `:cloud`), so the chat no longer defaults to a cloud model that silently returns nothing.
+- **ollama controls** — `GET /coach/api/ollama/status` + `POST /coach/api/ollama/{start,stop}`
+  start/stop the local `ollama serve` and report its state; `↻ models` re-lists.
+
+For a remote vLLM-ROCm endpoint instead, set `MINDXTRAIN_BACKEND=vllm` +
+`MINDXTRAIN_VLLM_BASE_URL`; the same streaming chat works against it
+(see [HANDOFF.md](HANDOFF.md) §§ 5–6).

docs/dcoach.md

@@ -0,0 +1,99 @@
+# dcoach — prove a CPU-trained model recalls its training
+
+`dcoach` is the decentralized-aware extension of the [Coach](coach.md). It closes
+mindXtrain's founding loop: **author a dataset → imprint a persona on a tiny model
+(CPU) → prove the model recalls the training → let governance rule on it → feed the
+verdict back into autotune.** It is also the on-ramp to the 2026 decentralized-training
+landscape (see [the deep dive](decentralized-training-deep-dive-2026.md)).
+
+Open it at **`/coach/dcoach`** (linked from the Coach header).
+
+## The proof loop
+
+```
+persona + skills ─► script.jsonl ─► imprint-train (trl_local, CPU)
+                                          │
+            ┌─────────────────────────────┘
+            ▼
+   probe recall  ──► classroom (before vs after)  ──► boardroom (rule)  ──► feedback
+   (base vs adapter)   recall ↑? persona kept?        approve / reject       tune next run
+```
+
+1. **Author** — a persona (e.g. `codephreak`) plus optional skills (software engineer,
+   platform architect, bash, solidity) is composed into chat rows
+   (`data/scripts.py::build_script_rows`). Each row carries the persona **system prompt**
+   + a user→assistant turn.
+2. **Imprint-train** — a tiny actor (default `HuggingFaceTB/SmolLM2-135M`) is LoRA-trained
+   on the script on the CPU lane (`train/backend_trl_cpu.py::run_trl_local`). The autotune
+   plan is frozen AOT — no JIT autotune in the loop.
+3. **Probe recall** — `eval/imprint.py::probe_recall` generates the actor's answer to each
+   inquiry **before** (base model) and **after** (base + adapter). The probe prepends the
+   *same persona system prompt the adapter trained under*, so the comparison measures what
+   the imprint actually learned rather than penalising a missing conditioning turn.
+4. **Classroom** — `governance/classroom.py::evaluate_classroom` scores before vs after
+   against the persona baseline (clean-room [llama-style evaluators](#clean-room-eval-tools)):
+   recall up? persona maintained? `passed = persona_maintained and pairwise ≥ 0.5`.
+5. **Boardroom** — the classroom graduation becomes a motion; a board (any-N, preset or
+   model-backed) rules **approve / reject**. A disputed board is settled by a prime-sized
+   **dojo**.
+6. **Feedback** — `autotune/feedback.py` records `(run_id, params, classroom_score,
+   outcome)` to an append-only ledger and `suggest_next_params` nudges the next run: a weak
+   or rejected imprint trains harder (more epochs, `grad_accum=1`); a clean pass holds.
+   `suggest_from_history` feeds the nudge back into `derive_training_params`.
+
+The whole chain is `governance/proof_loop.py::run_proof_loop`, streamed phase-by-phase to
+the UI via **`POST /coach/api/dcoach/run`** (SSE). It is heavy (real CPU training +
+generation) — expect a few minutes per run.
+
+## Clean-room eval tools
+
+`eval/llama_evals.py` reimplements the *behaviour* of LlamaIndex's evaluators (MIT) from
+their public contract — never copied. Each returns an `EvalScore{score∈[0,1], passing,
+reasoning, method}`:
+
+| Evaluator | What it measures | Backed by |
+|-----------|------------------|-----------|
+| `SemanticSimilarityEvaluator` | embedding/lexical closeness of two texts | `eval/imprint.py::_voice_similarity` |
+| `CorrectnessEvaluator` | response vs reference (LLM judge, 1–5 → [0,1]) | `governance/panel.chat_once` |
+| `PairwiseEvaluator` | after-utterance better than before toward the persona | judge (A/B/TIE) |
+| `GuidelineEvaluator` | rubric/agenda compliance | LLM judge |
+
+Endpoints: `POST /coach/api/classroom/evaluate`, `POST /coach/api/eval/prompt`,
+`POST /coach/api/autotune/feedback`.
+
+## Prompt tools — test cheap, promote if it wins
+
+**`/coach/prompts`** treats prompting as the cheapest pseudo-training: craft a system
+prompt + few-shot demonstrations, run them against a base model (streaming, **no
+training**), evaluate the outcome with the eval tools, and only if it's advantageous
+**make it permanent** by baking the prompt + demonstrations into an Ollama Modelfile
+(`POST /coach/api/modelfile/create`). Non-permanent experiment → promote on results.
+
+## How mindXtrain fits decentralized training
+
+The dcoach page renders a read-only panel (`GET /coach/api/decentralized`) mapping each
+2026 network to where mindXtrain plugs in. mindXtrain **does not mine** on any of them —
+every one is CUDA-first / hardware-gated. Instead it exposes a *verifiable, payable*
+training surface compatible with their verification primitives:
+
+| mindXtrain primitive | Maps to |
+|----------------------|---------|
+| AOT-only autotune plan (bit-reproducible run) | Gensyn **Verde + RepOps** training verification |
+| BLAKE3 verifiable receipt (`mindxtrain receipt`) | TOPLOC / checkpoint-hash verification; Templar **Gauntlet** auditing |
+| x402-metered training job | Per-job crypto metering — unbuilt territory across all networks |
+| AgenticPlace / ERC-8004 registration | Pluralis unextractable-model ownership / on-chain attribution |
+
+Networks covered: **Prime Intellect** (open stack, RL post-training), **Templar · Bittensor
+SN3** (Covenant-72B, the only live incentivized training market), **Nous · Psyche**
+(DisTrO on Solana), **Gensyn** (verification-first, Verde — the closest match), **Pluralis ·
+Node0** (model-parallel over WAN, unextractable models). Full analysis in
+[decentralized-training-deep-dive-2026.md](decentralized-training-deep-dive-2026.md) and
+[mindxtrain-llm-training-landscape-2026.md](mindxtrain-llm-training-landscape-2026.md).
+
+## Why this matters
+
+This is mindXtrain's **first-run proof**: that a model trained on the CPU lane actually
+*recalls* what it was trained on — measured, ruled on, and fed back, not asserted. It is
+also the bridge to the [mindX self-training loop](../README.md): the same loop that imprints
+`codephreak` here consumes the `machine.dream` corpus to produce the small model mindX falls
+back to.

docs/decentralized-training-deep-dive-2026.md

@@ -0,0 +1,182 @@
+# Decentralized Training: The Complete Landscape (Mid-2026)
+
+**A deep-dive companion to the mindXtrain training-stack survey** · Compiled June 2026
+
+---
+
+## TL;DR
+
+- **Decentralized training crossed its credibility threshold in 2025–2026.** Three landmark proofs: Templar's **Covenant-72B** (March 10, 2026 — 72B params, ~1.1T tokens, 70+ permissionless nodes over commodity internet, MMLU 67.1, ~LLaMA-2-70B class), Nous **Psyche/Consilience-40B** (largest internet pre-training run by parameter×token scale, coordinated on Solana), and Pluralis **Node0-7.5B** (first public *model-parallel* internet pretraining: 1,642 GPUs, 300+ participants, 198 cities, 36B tokens in 3 weeks).
+- **The algorithmic unlock is communication compression**: DiLoCo-family infrequent synchronization (~500× less communication), Streaming DiLoCo (two orders of magnitude bandwidth reduction), SparseLoCo (top-k sparsification + 2-bit quantization to 1–3% density, ~97% gradient compression — what powered Covenant-72B), DeMo/DisTrO (DCT + top-k momentum decoupling, up to 85× less data per GPU), and Pluralis Protocol Models (99% activation compression enabling model parallelism over WAN).
+- **Honest counterweight:** Prime Intellect — the most-funded name in the space — trained its flagship INTELLECT-3 (106B MoE) on a *centralized* 512×H200 cluster, a telling signal that for frontier-quality RL post-training, centralized still wins on engineering economics. RL post-training is the most decentralization-friendly workload; full frontier-scale pretraining over WAN remains unproven above ~100B dense.
+- **For mindXtrain:** the AOT-only reproducibility discipline is *precisely* the property that verification protocols (Gensyn Verde/RepOps, TOPLOC) require. The natural integration is RL-Swarm-style participation plus an x402-payable training-job surface with checkpoint-hash verification.
+
+---
+
+## 1. The Algorithms: How Training Escaped the Datacenter
+
+The core problem: datacenter training assumes NVLink/InfiniBand (100s of GB/s); the internet gives you 100–1000× less. Every viable approach attacks communication volume.
+
+### DiLoCo family (data-parallel, low-communication)
+- **DiLoCo** (DeepMind, [arXiv:2311.08105](https://arxiv.org/abs/2311.08105)) — the foundational recipe: a variant of federated averaging where each worker runs many local AdamW steps (H = hundreds), then synchronizes "pseudo-gradients" via an outer Nesterov-momentum optimizer. On C4, 8 workers matched fully synchronous training while **communicating 500× less**.
+- **Streaming DiLoCo** ([arXiv:2501.18512](https://arxiv.org/abs/2501.18512)) — three upgrades: synchronize parameter *subsets* in sequence (slashing peak bandwidth), overlap communication with continued training, and quantize exchanged data. Result: billion-scale training at matching quality with **two orders of magnitude less bandwidth**. This is the blueprint for cross-datacenter training (and the suspected basis of Google's multi-campus Gemini training).
+- **OpenDiLoCo** (Prime Intellect, [arXiv:2407.07852](https://arxiv.org/abs/2407.07852), [github.com/PrimeIntellect-ai/OpenDiLoCo](https://github.com/PrimeIntellect-ai/OpenDiLoCo)) — the open implementation (Hivemind-based), demonstrated at 1B+ across 3 countries at 90–95% utilization; scaled in INTELLECT-1 to 10B with int8 pseudo-gradients (~400× communication reduction, [arXiv:2412.01152](https://arxiv.org/abs/2412.01152)).
+- **SparseLoCo** (Templar/Bittensor, [arXiv:2508.15706](https://arxiv.org/abs/2508.15706)) — the 2026 state of the art for data-parallel WAN pretraining: error-feedback accumulators + **Top-k sparsification + 2-bit quantization reaching 1–3% density**, which *outperforms* DiLoCo baselines on loss while compressing ~97%+. Key insight: outer momentum can be locally approximated by the error-feedback buffer, and sparse aggregation can actually *improve* performance. This is what trained Covenant-72B over home internet connections.
+
+### Momentum-decoupling (Nous lineage)
+- **DeMo — Decoupled Momentum Optimization** ([arXiv:2411.19870](https://arxiv.org/abs/2411.19870), [github.com/bloc97/DeMo](https://github.com/bloc97/DeMo)) — drop-in replacement for momentum optimizers: decouple local momentum, apply a fast DCT transform + top-k sparsification, reuse momentum as error feedback. **Up to 85× less data per GPU** than AdamW-DDP at comparable loss (shown at 300M/1B); topology-agnostic, works over plain Ethernet.
+- **DisTrO** (Nous Research, [github.com/NousResearch/DisTrO](https://github.com/NousResearch/DisTrO)) — the productionized family built on DeMo's ideas, reducing inter-GPU transfer by several orders of magnitude; the engine of the Psyche network.
+
+### Model-parallel over WAN (Pluralis)
+- **SWARM Parallelism** ([arXiv:2301.11913](https://arxiv.org/abs/2301.11913)) — the precursor: pipeline-parallel training on unreliable, heterogeneous, low-bandwidth nodes (1.3B GPT over ~200Mb/s links with ~2× slowdown).
+- **Protocol Models** (Pluralis, [arXiv:2506.01260](https://arxiv.org/abs/2506.01260)) — the breakthrough for *model* parallelism: unlike data-parallel (exchange weight gradients), model-parallel must compress **activations and activation gradients** flowing between layers. Pluralis confines them to a predefined low-dimensional subspace exploited via the transformer's recursive structure, achieving **up to 99% compression with no convergence degradation**. Side effect with economic teeth: no participant ever holds full model weights — the model becomes an unextractable, protocol-native asset ("Unextractable Protocol Models"). Follow-on work: async pipeline-parallel with Nesterov stale-update correction, and >95% compression for context parallelism. Blog: [pluralis.ai/blog](https://pluralis.ai/blog/).
+
+### Other lineages
+- **Federated learning** (FedAvg lineage) — the ancestor of all of this; Flower Labs ([github.com/adap/flower](https://github.com/adap/flower)) carries it forward with FlowerLLM/Photon for federated LLM pretraining.
+- **Hivemind** ([github.com/learning-at-home/hivemind](https://github.com/learning-at-home/hivemind), MIT) — the P2P substrate (DHT, decentralized averaging, NAT traversal) under OpenDiLoCo, Petals, and Gensyn RL Swarm.
+- **Petals** ([github.com/bigscience-workshop/petals](https://github.com/bigscience-workshop/petals)) — collaborative inference/fine-tuning of 100B+ models, BitTorrent-style layer hosting.
+
+**Rule of thumb on bandwidth:** dense DDP needs ~GB/s-class links; DiLoCo-class needs ~100 Mb/s–1 Gb/s with minutes-scale sync windows (8-bit DiLoCo measured ~8.3 min all-reduce at 14 nodes); SparseLoCo/DeMo push viable participation down to consumer broadband.
+
+---
+
+## 2. The Networks: Who's Actually Training What
+
+### Prime Intellect — the open superintelligence stack
+- **Track record:** INTELLECT-1 (10B, OpenDiLoCo, 3 continents) → **INTELLECT-2** (32B, first globally distributed RL run; [arXiv:2505.07291](https://arxiv.org/abs/2505.07291)) → **INTELLECT-3** (106B MoE, 12B active, SFT+RL on GLM-4.5-Air base; [arXiv:2512.16144](https://arxiv.org/abs/2512.16144), [huggingface.co/PrimeIntellect/INTELLECT-3](https://huggingface.co/PrimeIntellect/INTELLECT-3), released Nov 27, 2025 — best-in-class math/code/reasoning for its size).
+- **The catch:** INTELLECT-3 was trained on a **centralized 512× H200 cluster**, not the decentralized protocol — a candid pivot toward "open-source models + compute platform" over pure decentralization ([blog](https://www.primeintellect.ai/blog/intellect-3), critical coverage: [implicator.ai](https://www.implicator.ai/prime-intellects-intellect-3-open-source-ambition-meets-centralized-reality/)).
+- **Platform (2026):** **Lab** ([blog](https://www.primeintellect.ai/blog/lab)) unifies the Environments Hub, hosted RL training, and hosted evals — 10,000+ training jobs run by hundreds of teams; opened fully May 2026. Joined the NVIDIA Nemotron coalition (June 2026). Compute Exchange aggregates global GPU supply.
+- **Protocol/token:** peer-to-peer compute protocol live on internal testnet (powered SYNTHETIC-2 and INTELLECT-2); contracts on Base Sepolia with a RewardsDistributor pattern suggesting an eventual token; no token launched as of June 2026.
+- **Repos (all permissive):** [prime-rl](https://github.com/PrimeIntellect-ai/prime-rl) · [protocol](https://github.com/PrimeIntellect-ai/protocol) · [toploc](https://github.com/PrimeIntellect-ai/toploc) · [shardcast](https://github.com/PrimeIntellect-ai/shardcast) · [verifiers](https://github.com/PrimeIntellect-ai/verifiers) · [OpenDiLoCo](https://github.com/PrimeIntellect-ai/OpenDiLoCo)
+- **Funding:** $20M+ total — Founders Fund lead, with Karpathy, Delangue, Dylan Patel, Tri Dao, Emad Mostaque among angels.
+
+### Nous Research / Psyche — DisTrO on Solana
+- **Psyche** ([github.com/PsycheFoundation/psyche](https://github.com/PsycheFoundation/psyche), **Rust, Apache 2.0**; [docs](https://nousresearch.com/nous-psyche)) — decentralized training network with **coordination on Solana** for fault-tolerant, censorship-resistant orchestration; compute off-chain running DisTrO-compressed training.
+- **Consilience-40B:** dense 40B with DeepSeek-style MLA attention, target ~20T tokens (FineWeb 14T + FineWeb-2 4T + Stack v2 upsampled to 1T) — by parameters×tokens, **the largest distributed pre-training run ever** over the internet; deliberately sized to train on one HGX and infer on a 3090. As of the ["Next Phase of Psyche"](https://nousresearch.com/the-next-phase-of-psyche) (Nov 2025), the testnet run validated internet-bandwidth training at scale and Psyche pivoted to training multiple models in parallel.
+- **Token status (important):** as of April 2026 **no official Nous/Psyche token exists** — "NOUS" pairs on Solana DEXs are unofficial; don't confuse with Nosana ($NOS).
+- **Funding:** $50M from Paradigm at ~$1B valuation.
+- **Models:** Hermes series ([huggingface.co/NousResearch](https://huggingface.co/NousResearch)) validates the open-model credibility that underwrites the network.
+
+### Gensyn — verification-first ML compute protocol
+- **Architecture:** four primitives — execution, verification, communication, coordination — on a custom Ethereum-rollup testnet. Backed by a16z ($43M Series A).
+- **Status (2026):** RL Swarm (peaked ~12,000 testnet nodes; later environments: CodeZero coding swarm) and BlockAssist/CodeAssist have been **paused/sunset**; focus consolidated on **Delphi**, a "prediction market for machine intelligence," as the first Mainnet application. **Mainnet not yet launched** as of June 2026; testnet docs: [docs.gensyn.ai/testnet](https://docs.gensyn.ai/testnet).
+- **Repos:** [rl-swarm](https://github.com/gensyn-ai/rl-swarm) · [rl-swarm-contracts](https://github.com/gensyn-ai/rl-swarm-contracts) · [repops-demo](https://github.com/gensyn-ai/repops-demo). RL Swarm hardware floor was deliberately low: arm64/x86 CPU + 32GB RAM, or NVIDIA 3090/4090/5090/A100/H100; macOS, Linux, Windows-WSL2; Python 3.10–3.13.
+- **Research:** Verde ([arXiv:2502.19405](https://arxiv.org/abs/2502.19405)), SAPO swarm-sampling policy optimization, NoLoCo (no-all-reduce training, [arXiv:2506.10911](https://arxiv.org/abs/2506.10911)), Gauntlet-style contribution scoring lineage.
+
+### Templar / Bittensor SN3 — the permissionless proof
+- **Covenant-72B** (announced March 10, 2026): **72B params, ~1.1T tokens, 70+ independent miners, fully permissionless** — anyone with GPUs could join/leave mid-run — over commodity internet. MMLU 67.1 (~LLaMA-2-70B class). Enabled by **SparseLoCo** (146× communication reduction claimed via sparsification + 2-bit quantization + error feedback) and the **Gauntlet** contributor-scoring system (loss-based evaluation of each node's submitted updates, with TAO/alpha incentives and slashing-style penalties for junk contributions). Apache-licensed model.
+- **Ecosystem effects:** τemplar token +194% in a week; TAO ~+30–40%; Jensen Huang likened it to "folding@home for AI"; coverage from Jack Clark's Import AI. Bittensor in March 2026: ~128 active subnets, TAO ~$3.4B market cap, subnet alpha tokens ~$1.4B combined (see [arXiv risk study](https://arxiv.org/pdf/2603.29751)).
+- **Links:** [tplr.ai](https://tplr.ai) · [github.com/tplr-ai/templar](https://github.com/tplr-ai/templar) · Bittensor: [github.com/opentensor/bittensor](https://github.com/opentensor/bittensor) · related training subnets: Macrocosmos IOTA ([macrocosmos.ai](https://www.macrocosmos.ai)) for pipeline-parallel pretraining experiments.
+- **dTAO mechanics:** each subnet has its own alpha token bonded against TAO; miners earn by validator-scored contribution quality — the only live, fully incentivized, permissionless training market as of mid-2026.
+
+### Pluralis Research — Protocol Learning (model parallel)
+- **Node0-7.5B** ([dashboard.pluralis.ai](https://dashboard.pluralis.ai), [github.com/PluralisResearch/node0](https://github.com/PluralisResearch/node0)): the first public **model-parallel** internet pretraining run — completed after **36B tokens over 3 weeks with 300+ active participants and 1,642 GPUs across 198 cities**, joinable with a single 16GB consumer GPU (3090-class). Built on Protocol Models compression ([arXiv:2506.01260](https://arxiv.org/abs/2506.01260)).
+- **Strategic differentiator:** weights are sharded such that **no participant can extract the full model** — enabling on-protocol model ownership, revenue attribution, and access gating (deeply relevant to DAIO-style on-chain asset thinking). Funding: $7.6M seed (USV, CoinFund).
+
+### Others worth tracking
+- **Flower Labs** ([flower.ai](https://flower.ai), [github.com/adap/flower](https://github.com/adap/flower), Apache 2.0) — federated LLM training (FlowerLLM; Photon system paper) with the largest federated-learning developer community.
+- **Exo Labs** ([github.com/exo-explore/exo](https://github.com/exo-explore/exo)) — cluster your own heterogeneous consumer devices (Macs, mining rigs) for local training/inference; not a token network.
+- **Petals / Hivemind** — see §1; research substrate more than incentive network.
+- **Compute marketplaces (supply side, not training protocols):** Akash ([akash.network](https://akash.network), AKT), io.net (IO), Render, Aethir, Spheron — these price raw GPU hours; training networks sit a layer above.
+- **FedML/TensorOpera, Bagel** ([bagel.net](https://bagel.net) — "Bakery" fine-tuning marketplace research) — earlier-stage or pivoted.
+
+---
+
+## 3. Verification: The Trust Layer
+
+(Extends the verification section of the main survey — repos there remain canonical.)
+
+| Approach | System | Verifies | Production status |
+|---|---|---|---|
+| Activation LSH | [TOPLOC](https://github.com/PrimeIntellect-ai/toploc) ([arXiv:2501.16007](https://arxiv.org/abs/2501.16007)) | Inference/rollouts | Used in INTELLECT-2 |
+| Refereed delegation + bitwise-reproducible ops | Verde + RepOps ([arXiv:2502.19405](https://arxiv.org/abs/2502.19405), [repops-demo](https://github.com/gensyn-ai/repops-demo)) | **Training** steps | Gensyn testnet |
+| Optimistic fraud proofs | [opML](https://github.com/ora-io/opml) ([arXiv:2401.17555](https://arxiv.org/abs/2401.17555)) | Inference (training targeted) | ORA on-chain AI |
+| zkML | [EZKL](https://github.com/zkonduit/ezkl), [ddkang/zkml](https://github.com/ddkang/zkml), Lagrange DeepProve | Small-model inference proofs | Niche; cost-bound |
+| Economic scoring | Templar **Gauntlet** (loss-evaluation of contributions + token slashing) | Training contributions statistically | **Live, incentivized** (SN3) |
+| TEEs | NVIDIA Confidential Computing (H100), Intel TDX, AWS Nitro | Execution environment | Growing in compute markets |
+
+**The honest state:** cryptographic verification of *pretraining* at scale remains unsolved in production. Templar's Gauntlet shows the pragmatic alternative — statistical/economic verification (does your update reduce loss?) backed by stake. Verde/RepOps is the most principled training-verification design but needs deterministic execution — which is exactly what an **AOT-only artifact policy** provides. zkML proof costs are still orders of magnitude above native compute for LLM-scale work.
+
+---
+
+## 4. Why RL Is the Decentralization Sweet Spot
+
+- RL post-training = **embarrassingly parallel rollout generation** (inference-heavy, communication-light) + a small trainer. INTELLECT-2's architecture is the template: [prime-rl](https://github.com/PrimeIntellect-ai/prime-rl) async trainer ← TOPLOC-verified rollouts from untrusted inference nodes ← [shardcast](https://github.com/PrimeIntellect-ai/shardcast) weight broadcasts.
+- Gensyn's RL Swarm generalized this into multi-agent collaborative RL (answer/critique/revise games; SAPO swarm sampling), demonstrating swarm-trained models learn faster than solo — and that heterogeneous, consumer hardware can contribute usefully because rollouts don't need gradient sync.
+- The **environments economy** is the new commodity layer: Prime Intellect's Environments Hub + [verifiers](https://github.com/PrimeIntellect-ai/verifiers) library, Gensyn CodeZero, reasoning-gym lineage. Whoever owns high-quality verifiable environments owns RL training demand. (For PYTHAI: blockchain task environments — Foundry test-passing, contract auditing, x402 flow completion — are an unclaimed niche.)
+- Caveat from the main survey still holds: decentralized RL gains concentrate in trained domains (math/code); broad transfer lags centralized RL.
+
+---
+
+## 5. Economics & Crypto Integration
+
+- **Live token economics:** only Bittensor — TAO emission split across 128 subnets via dTAO; subnet alpha tokens (τemplar) reprice on demonstrated capability. Covenant-72B was the first event where a training result directly repriced a token 194%.
+- **Pending:** Prime Intellect (Base testnet contracts, RewardsDistributor pattern, no token), Gensyn (testnet points → expected token at Mainnet; Delphi first), Psyche (Solana-coordinated, explicitly **no official token yet** as of April 2026 — beware impostor "NOUS" pairs).
+- **Funding landscape:** a16z→Gensyn ($43M), Paradigm→Nous ($50M @ ~$1B), Founders Fund→Prime Intellect ($20M+), USV/CoinFund→Pluralis ($7.6M); DCG's Yuma accelerates Bittensor ecosystem; Grayscale holds TAO.
+- **Sober read:** the only mechanism so far proven to incentivize *useful* training (not speculation) is Templar's loss-scored, slashing-backed contribution market. Everything else either pays points (Gensyn), pays nothing yet (Psyche, Pluralis Node0 — reputational/dashboard credit), or routes around tokens entirely (Prime Intellect's fiat compute exchange).
+- **x402 relevance:** none of these networks natively meter per-job crypto payments; a per-training-job x402 paywall (Algorand x402-avm "Parsec" in your stack) in front of a verifiable training endpoint is genuinely unbuilt territory.
+
+---
+
+## 6. Hardware & Network Realities
+
+- **Demonstrated efficiency:** INTELLECT-1 hit 83–96% utilization (14 nodes, 3 continents); OpenDiLoCo 90–95%; Pluralis cites GPT-1.3B pipeline-parallel over 200Mb/s at ~2× slowdown; SparseLoCo makes consumer broadband viable at 72B. Expect 1.2–3× wall-clock penalty vs an equivalent co-located cluster when the algorithm fits, far worse when it doesn't.
+- **Consumer hardware floors:** Pluralis Node0 — single 16GB GPU (3090); Gensyn RL Swarm — even CPU+32GB RAM; Templar mining — prosumer multi-GPU favored; Psyche — 3090-class inference target, training nodes larger.
+- **AMD/ROCm reality check:** every major network's node software is **NVIDIA/CUDA-first** (Gensyn lists 3090/4090/5090/A100/H100; Pluralis requires CUDA ≤12.x). MI300X participation today means either contributing through GPU marketplaces (Prime Intellect Compute Exchange lists heterogeneous supply) or running protocol-side/trainer-side infrastructure rather than mining. This is a gap — and an opening for ROCm-native node ports.
+- **Networking stacks:** Hivemind DHT (+ relays/NAT traversal) dominates (OpenDiLoCo, Petals, RL Swarm); Psyche uses Solana for coordination + P2P data plane (iroh-class Rust networking); Templar uses Bittensor's axon/dendrite gossip + object storage for gradient exchange.
+- **Churn tolerance:** all serious systems assume nodes join/leave mid-run — DiLoCo's infrequent sync, SWARM's stochastic rewiring, Gauntlet's per-contribution scoring, and Psyche's on-chain checkpointing all exist precisely for this.
+
+---
+
+## 7. Critical Assessment & Open Problems
+
+1. **Scale ceiling:** largest decentralized pretraining = 72B dense / ~1.1T tokens (Covenant). Frontier centralized runs are training 10×+ larger models on 50×+ tokens with 100,000+ GPU clusters. The gap is closing on a log scale, not disappearing.
+2. **The Prime Intellect signal:** when the best-funded decentralized lab trains its flagship centrally (512×H200) while open-sourcing the stack, the message is: decentralization currently wins on *access and sovereignty*, not on cost or speed at frontier quality.
+3. **Verification gap:** pretraining verification is economic, not cryptographic. A motivated adversary inside a permissionless run is mitigated (Gauntlet slashing, Byzantine-robust aggregation), not eliminated. Data poisoning in permissionless data-parallel runs remains under-studied.
+4. **Model parallelism over WAN** is the frontier — Pluralis is essentially alone in production here; if Protocol Models scales past ~10B with heterogeneous consumer cards, the "no single node has the weights" property changes the ownership game entirely.
+5. **Regulatory horizon:** the "no-off problem" ([arXiv:2412.07890](https://arxiv.org/pdf/2412.07890)) — once training is a protocol, no one can stop it. Expect compute-governance and export-control attention as runs approach frontier capability.
+6. **Forecast:** decentralized *post-training* (RL, fine-tuning, distillation) reaches economic parity first — arguably already there for verifiable-reward domains. Decentralized *pretraining* plausibly reaches 100B+ dense / multi-trillion tokens by 2027 via SparseLoCo-class compression + dTAO-class incentives, but frontier parity requires either an algorithmic surprise or centralized-compute commoditization.
+
+---
+
+## 8. Practical Integration for mindXtrain / PYTHAI
+
+**Participate (today, ranked by fit):**
+1. **Prime Intellect Lab / Environments Hub** — publish blockchain-native RL environments (Foundry-test-passing, Solidity audit, Algorand x402 flows) via the [verifiers](https://github.com/PrimeIntellect-ai/verifiers) library; train against them with hosted RL or your own prime-rl deployment. Lowest friction; AMD-agnostic since you consume the platform.
+2. **Templar SN3 mining** ([github.com/tplr-ai/templar](https://github.com/tplr-ai/templar)) — the only incentivized live training market; NVIDIA prosumer hardware; real TAO/alpha yield, real slashing risk.
+3. **Pluralis Node0-class events** ([github.com/PluralisResearch/node0](https://github.com/PluralisResearch/node0)) — 16GB+ NVIDIA GPU, port 49200 exposed, Docker; watch for the next run.
+4. **Psyche** ([github.com/PsycheFoundation/psyche](https://github.com/PsycheFoundation/psyche)) — Rust/Apache-2.0, Solana coordination (your chain-stack adjacency is an advantage); contribution currently reputational.
+5. **Gensyn** — RL Swarm paused; watch Delphi → Mainnet for the token-incentivized restart.
+
+**Build (the mindXtrain thesis):**
+- Your **AOT-only discipline is the verification primitive**: deterministic compiled artifacts + pinned ROCm/libtorch = exactly the bitwise-reproducibility Verde/RepOps demands. A mindXtrain node that ships its AOT probe artifact alongside checkpoint hashes is *natively verifiable*.
+- **Training-as-a-service with x402:** front a prime-rl or Axolotl/TRL pipeline with an x402-metered endpoint (Parsec on Algorand from your stack); escrow per-job payment against TOPLOC-style rollout proofs or Verde-style checkpoint-hash spot-checks; settle on completion. Register the service as an ERC-8004 agent on AgenticPlace. Nobody has shipped this combination.
+- **ROCm node ports** of rl-swarm / node0 / psyche clients are an open contribution lane with outsized visibility — every network is CUDA-locked and knows it.
+
+---
+
+## Network Comparison Table
+
+| Network | Algorithm | Largest demonstrated | Verification | Token (Jun 2026) | Min hardware | Code (license) |
+|---|---|---|---|---|---|---|
+| Prime Intellect | OpenDiLoCo → prime-rl async RL | INTELLECT-2 32B RL (decentralized); INTELLECT-3 106B MoE (centralized) | TOPLOC | None (Base testnet contracts) | Platform consumer / any | [PrimeIntellect-ai](https://github.com/PrimeIntellect-ai) (Apache 2.0) |
+| Nous Psyche | DisTrO/DeMo, Solana coordination | Consilience-40B @ 20T-token target | Solana-anchored checkpoints | None official (beware fakes) | Prosumer GPU+ | [PsycheFoundation/psyche](https://github.com/PsycheFoundation/psyche) (Apache 2.0) |
+| Gensyn | RL Swarm (Hivemind), NoLoCo, SAPO | ~12K-node RL swarm (testnet) | Verde + RepOps | Testnet points; token at Mainnet | CPU+32GB RAM or 3090+ | [gensyn-ai](https://github.com/gensyn-ai) (varied OSS) |
+| Templar (SN3) | SparseLoCo + Gauntlet | **Covenant-72B, 1.1T tokens, permissionless** | Economic (loss-scored, slashed) | **Live**: TAO + τemplar alpha | Prosumer/multi-GPU NVIDIA | [tplr-ai/templar](https://github.com/tplr-ai/templar) (MIT) |
+| Pluralis | Protocol Models (model-parallel, 99% compression) | Node0-7.5B: 1,642 GPUs, 198 cities | Weight-sharding (unextractable) | None | 16GB GPU (3090) | [PluralisResearch/node0](https://github.com/PluralisResearch/node0) |
+| Flower | Federated (Photon/FlowerLLM) | Federated LLM pretraining research | — | None | Any | [adap/flower](https://github.com/adap/flower) (Apache 2.0) |
+
+---
+
+## Key Papers Index
+
+[DiLoCo 2311.08105](https://arxiv.org/abs/2311.08105) · [Streaming DiLoCo 2501.18512](https://arxiv.org/abs/2501.18512) · [OpenDiLoCo 2407.07852](https://arxiv.org/abs/2407.07852) · [SparseLoCo 2508.15706](https://arxiv.org/abs/2508.15706) · [DeMo 2411.19870](https://arxiv.org/abs/2411.19870) · [SWARM 2301.11913](https://arxiv.org/abs/2301.11913) · [Protocol Models 2506.01260](https://arxiv.org/abs/2506.01260) · [INTELLECT-1 2412.01152](https://arxiv.org/abs/2412.01152) · [INTELLECT-2 2505.07291](https://arxiv.org/abs/2505.07291) · [INTELLECT-3 2512.16144](https://arxiv.org/abs/2512.16144) · [TOPLOC 2501.16007](https://arxiv.org/abs/2501.16007) · [Verde 2502.19405](https://arxiv.org/abs/2502.19405) · [opML 2401.17555](https://arxiv.org/abs/2401.17555) · [NoLoCo 2506.10911](https://arxiv.org/abs/2506.10911) · [No-Off Problem 2412.07890](https://arxiv.org/pdf/2412.07890)
+
+---
+
+## Caveats
+
+- Token prices, node counts, and network statuses shift weekly; figures here are snapshots from announcements and coverage through early June 2026.
+- Covenant-72B performance claims (LLaMA-2-70B parity, 146× compression) originate from the Templar team and secondary coverage; independent replication of the full run is not yet published.
+- Several "largest ever" claims (Psyche Consilience vs Covenant) measure different things — parameters, tokens processed, or parameters×tokens — and both teams claim records under their preferred metric.
+- AMD/ROCm support statements reflect documented requirements as of writing; check each repo's README before provisioning hardware.

docs/development.md

@@ -0,0 +1,330 @@
+# Development workflow
+
+Conventions and invariants for working in this repo. Read once before opening a PR.
+
+## Toolchain
+
+- **Python 3.12** (`>=3.12,<3.13`) — pinned; matches `rocm/primus:v26.2`.
+- **uv** — single project (no workspace). `uv sync` installs the base deps;
+  `uv sync --extra <group>` adds optional groups.
+- **ruff** — replaces black/isort/flake8/pyupgrade. Config in
+  [`pyproject.toml`](../pyproject.toml).
+- **mypy --strict** — only on `mindxtrain/config` and `mindxtrain/provenance`
+  (the schemas + manifest paths). Training / eval code is exempt.
+- **pytest** + `pytest-asyncio` — fast unit tests; GPU tests are manual on
+  the MI300X.
+- **Foundry** — Solidity contracts in `contracts/`. Installed on the MI300X
+  droplet for the on-chain anchoring path.
+
+## Optional dependency groups
+
+`pyproject.toml` defines six `[project.optional-dependencies]` groups:
+
+| Group   | Adds                                                        |
+|---------|-------------------------------------------------------------|
+| `ml`    | trl, transformers, peft, accelerate, datasets               |
+| `eval`  | lm-eval, lighteval, inspect-ai, jinja2                      |
+| `data`  | datasketch, sentence-transformers, faiss-cpu, pyarrow       |
+| `serve` | vllm                                                        |
+| `chain` | web3, py-algorand-sdk, huggingface-hub                      |
+| `obs`   | opentelemetry-sdk, prometheus-client, psutil                |
+
+Plus `all` which pulls everything except `amd-quark` (which ships in the
+rocm/primus container, see [HANDOFF.md](HANDOFF.md) §3).
+
+The base install (no extras) is enough for: the CLI, the Coach UI, the
+autotune dry-run, manifest verify, the operator FastAPI app, and every
+in-process Python utility (registry, hot-swap, agent loop, ContextManager,
+data filter, sequence packing). See
+[actualization_status.md](actualization_status.md) for the per-module map.
+
+## Lazy-import pattern
+
+Every module that wants an optional dep guards the import inside the
+function that needs it:
+
+```python
+def run_lm_eval(model_dir: Path, tasks: list[str]) -> Path:
+    if not _lm_eval_available():
+        msg = "lm-eval not installed; run `uv sync --extra eval`."
+        raise RuntimeError(msg)
+    ...  # subprocess wrap that uses the dep
+```
+
+Two implications:
+
+1. `import mindxtrain.eval.harness` always succeeds even without `--extra eval`.
+2. The error message includes the exact `uv sync --extra <group>` to run.
+
+This is the canonical pattern; new modules that take optional deps must
+follow it.
+
+## The standard local cycle
+
+```bash
+uv sync                                                    # base install
+uv run ruff check --fix .                                  # lint + auto-fix
+uv run mypy mindxtrain/config mindxtrain/provenance        # types where strict
+uv run pytest -q                                           # → 564 passed in ~5s
+```
+
+CI runs the same four commands on Ubuntu 24.04 / Python 3.12 (CPU-only).
+See [`.github/workflows/ci.yml`](../.github/workflows/ci.yml).
+
+## Repository layout
+
+```
+.
+├── pyproject.toml                # single project; optional-dep groups
+├── README.md                     # entry doc (the only root .md besides CLAUDE/AGENTS)
+├── CLAUDE.md, AGENTS.md          # agent-tooling entrypoints (required at root)
+├── NOTICE, LICENSE-*             # legal
+├── Containerfile, compose.yaml   # podman entry points
+├── docs/                         # all documentation (index: docs/NAV.md)
+│   ├── NAV.md                    # docs index
+│   ├── HANDOFF.md                # operator checklist
+│   ├── dcoach.md                 # the proof loop + decentralized fit
+│   ├── CHANGELOG.md
+│   └── …                         # architecture, coach, governance, decentralized, reference
+├── mindxtrain/                   # the package — 12 subpackages, ~99 modules
+│   ├── cli/                      # typer CLI (9 verbs)
+│   ├── config/                   # 10-section Pydantic schema + JSON defaults
+│   ├── data/                     # curate → dedupe → filter → tokenize → pack → synth → verify
+│   ├── models/                   # registry + chat templates + 5 base presets
+│   ├── train/                    # sft, dpo, grpo, rlhf, tool_use, distributed, callbacks, recipes/
+│   ├── eval/                     # lighteval / inspect_ai / bfcl / persona / agenda / card
+│   ├── autotune/                 # 60s AOT probe — the differentiator
+│   ├── operator/                 # FastAPI app, Coach UI, ml-intern patterns
+│   ├── storage/                  # local_fs / hf_hub / lighthouse / ipfs
+│   ├── provenance/               # manifest, hashing, verify, erc8004, algorand, x402
+│   ├── deploy/                   # registry, hot_swap, ab_test, vllm_launcher, quark
+│   └── budget/                   # ResourceBudget + cloud-provider stubs
+├── contracts/                    # Foundry workspace (ERC-8004 attestation)
+├── ops/                          # containerfiles, compose, k8s, vmm, gensyn
+├── examples/                     # demo YAMLs
+├── tests/                        # pytest — 566 tests (CPU-only smoke)
+└── docs/
+    ├── *.md                      # current state (this directory)
+    └── blueprints/               # source design briefs (frozen)
+```
+
+## Reuse boundaries
+
+- **From `/home/hacker/mindX/`** (production codebase): Codephreak persona
+  JSON loaded at runtime via `MINDXTRAIN_PERSONA_PATH`. Do not copy file
+  bytes — load via env var.
+- **Not** from `/home/hacker/aglm/` — broken per its own README. Use only
+  for reference to legacy class names mindxtrain2.md flagged as needing
+  refactor.
+
+## Invariants
+
+These are non-negotiable; violating them is a deployment bug, not a style
+preference.
+
+1. **AOT-only.** No `torch.compile(mode="max-autotune")` in production paths.
+   No JIT autotune in vLLM serving (set `VLLM_USE_TRITON_FLASH_ATTN=0` if
+   needed). The `autotune.policy: aot_only` field in the YAML is the
+   contract; tested at
+   `tests/test_config_schema.py::test_qwen3_8b_sft_lora_validates`.
+2. **`hardware.gpus: 1 | 8` only.** 2/4-GPU MI300X FSDP groups hit
+   asymmetric xGMI; the schema rejects them at parse time. Tested at
+   `tests/test_config_schema.py::test_xgmi_2gpu_rejected` and
+   `tests/test_distributed.py`.
+3. **Seven MI300X env vars in `train.env`** (defaults, can be overridden by
+   the autotune plan but never removed): `HSA_NO_SCRATCH_RECLAIM=1`,
+   `NVTE_CK_USES_BWD_V3=1`, `NVTE_CK_IS_V3_ATOMIC_FP32=1`,
+   `PRIMUS_TURBO_ATTN_V3_ATOMIC_FP32=1`, `NCCL_MIN_NCHANNELS=112`,
+   `HIP_FORCE_DEV_KERNARG=1`, `PYTORCH_ROCM_ARCH=gfx942`.
+4. **`extra: forbid` on every Pydantic model.** Unknown YAML keys raise
+   `ValidationError`. Tested at
+   `tests/test_config_schema.py::test_extra_field_forbidden`.
+5. **Configs are immutable once loaded** (`frozen: true`).
+6. **Solidity contracts: no proxies, no `Ownable`, no admin keys, no setters.**
+   `mindxtrain_registry.sol` is write-once. Rotating any parameter requires a
+   fresh deploy.
+7. **Lazy imports for optional deps** — see the pattern above.
+
+## Training lanes (CPU / local-GPU / MI300X)
+
+Three ways to actually run a fine-tune, selected by `train.backend`:
+
+| Lane | Backend | Device | When |
+|------|---------|--------|------|
+| CPU | `trl_cpu` | CPU, float32 (in-process TRL) | mindX self-training, smoke runs, no GPU |
+| Local GPU | `trl_local` | auto: CUDA/ROCm GPU (bf16/fp16) else CPU fallback | consumer Radeon RX / NVIDIA RTX, or a laptop |
+| MI300X | `axolotl`/`unsloth`/`torchtune`/`primus` | gfx942 subprocess + 7 env vars | the AOT MI300X target |
+
+`trl_local` is the **device-aware** in-process lane (`backend_trl_cpu.py::run_trl_local`):
+it picks the GPU when `torch.cuda.is_available()` (ROCm surfaces through the same API),
+else logs `no accelerator detected → CPU fallback` and runs on CPU. The same recipe
+(`mindx_fallback_qwen3_1_5b_local`) therefore runs unchanged on a gaming GPU or a laptop.
+`trl_cpu` is `run_trl_local(..., force_cpu=True)`; `MINDXTRAIN_FORCE_CPU=1` forces the
+fallback anywhere. The in-process lanes never inject the seven MI300X env vars.
+
+Confirm which device a box will use:
+```bash
+uv run python -c "import torch; print(torch.cuda.is_available(), torch.version.hip)"
+```
+
+**Unsupported:** integrated Vega/RDNA APUs (e.g. Ryzen "Raven"/`gfx90c`) are not ROCm
+targets and fall back to CPU. A discrete RX 6800/7900 (`gfx1030`/`gfx1100`) or any NVIDIA
+RTX is the intended consumer GPU.
+
+## Adding a new recipe
+
+1. Drop a YAML at `mindxtrain/train/recipes/<name>.yaml`. Validate locally:
+   ```bash
+   uv run python -c "from mindxtrain.config.loader import load_config; load_config('mindxtrain/train/recipes/<name>.yaml')"
+   ```
+2. The `tests/test_config_schema.py::test_all_recipes_validate` test will
+   pick it up automatically — re-run pytest.
+3. Add a row to [docs/yaml_schema.md](yaml_schema.md) only if the recipe
+   exercises a previously-unused field.
+
+## Adding a new training backend
+
+1. Add `mindxtrain/train/backend_<name>.py` exposing a
+   `run_<name>(cfg, plan, out_dir) -> Path` function (or for in-process TRL
+   trainers, a `run_<name>(cfg, out_dir) -> Path` function).
+2. Wire it into `mindxtrain/train/dispatch.py`'s `if backend == ...` ladder.
+3. Add `<name>` to the `TrainingBackend` literal in
+   `mindxtrain/config/schema.py`.
+4. Update [docs/cli.md](cli.md) "Where the verbs live" table.
+
+## Adding a new model backend (operator)
+
+1. Add `mindxtrain/operator/backends/<name>.py` with a `Backend` subclass
+   decorated `@register_backend("<name>")`.
+2. Side-effect import it from `mindxtrain/models/registry.py` so registration
+   runs on package import.
+3. Add a runtime branch in `mindxtrain/operator/app.py::chat_completions` for
+   the env-var-driven kwargs.
+
+## Adding a new training method
+
+1. Define a `_MethodBase` subclass in `mindxtrain/config/schema.py` with
+   `kind: Literal["<name>"] = "<name>"` and the method-specific fields.
+2. Add it to the `TrainMethod` discriminated union.
+3. Add a `mindxtrain/train/<name>.py` runner (TRL wrap or subprocess).
+4. Update the dispatch path so a YAML with `train.method.kind == "<name>"`
+   reaches the runner.
+5. Add a recipe under `mindxtrain/train/recipes/` exercising it.
+6. Update `docs/yaml_schema.md` "train.method" table.
+
+## Adding a new optional-dep group
+
+1. Add the entry to `[project.optional-dependencies]` in `pyproject.toml`.
+2. Add a row to the table in [actualization_status.md](actualization_status.md).
+3. Update [development.md](development.md) and [quickstart.md](quickstart.md).
+
+## Adding a new doc
+
+1. Write `docs/<name>.md`.
+2. Add a one-line entry to [`docs/NAV.md`](NAV.md) under the appropriate section.
+
+## Live training UI
+
+The Coach UI's "Train" step (`#step-train` in
+[`coach/static/index.html`](../mindxtrain/operator/coach/static/index.html))
+launches a training run and streams loss / lr / log lines back into the
+browser over Server-Sent Events. Architecture:
+
+- **Registry**: `mindxtrain.operator.runs.RunRegistry` is an in-memory
+  singleton (one per uvicorn process) keyed by `run_id`. Snapshots are
+  immutable `Run` records (frozen Pydantic); state changes produce new
+  snapshots via `model_copy`.
+- **Event schema**: `TrainEvent` is a tagged union over `StatusEvent`,
+  `StepEvent`, `EvalEvent`, `LogEvent`, `EnergyEvent` — all with
+  `extra="forbid", frozen=True`. Wire format: `event: <kind>\ndata:
+  <event.model_dump_json()>\n\n`.
+- **Two ingestion paths**, deduped by `(run_id, step)` in
+  `RunRegistry.publish`:
+  1. Subprocess stdout regex (`parse_trainer_log_line`) — works on the
+     base install, parses HF Trainer's `'loss': … 'learning_rate': …`
+     log lines.
+  2. In-process `mindxtrain.train.callbacks.StreamCallback` — POSTs to
+     `/coach/api/runs/{id}/ingest` (loopback only). Requires `--extra ml`.
+- **Subprocess orchestration**: `spawn_subprocess_streaming` uses
+  `subprocess.Popen(stdout=PIPE, bufsize=1, text=True)` and tees lines
+  to both `train.log` (the durable on-disk artifact) and
+  `RunRegistry.publish_threadsafe` from a daemon thread. We use
+  `Popen` (not `asyncio.create_subprocess_exec`, not `BackgroundTasks`)
+  so the child outlives the launch HTTP request and `SIGINT`-then-`SIGTERM`
+  cancellation matches the CLI Ctrl-C path.
+
+### Routes
+
+All under `/coach/api/runs`:
+
+| Verb | Path | Purpose |
+|---|---|---|
+| POST | `/launch` | Spawn a run; returns `Run` immediately. 503 if `accelerate` is missing. |
+| GET  | `/` | List active + last 20 runs. |
+| GET  | `/{id}` | `Run` snapshot. |
+| GET  | `/{id}/events` | SSE — all event kinds. Replays last 200 buffered on connect. |
+| GET  | `/{id}/logs` | SSE — `kind="log"` only. |
+| POST | `/{id}/cancel` | `SIGINT` then `SIGTERM` after grace. |
+| POST | `/{id}/ingest` | Loopback-only — used by `StreamCallback`. |
+
+SSE responses set `Cache-Control: no-cache`, `X-Accel-Buffering: no`,
+`Connection: keep-alive` so reverse proxies don't buffer the stream.
+
+### Invariants
+
+- `import mindxtrain.operator.runs` succeeds **without** `--extra ml`. The
+  in-process `StreamCallback` requires `transformers`; the subprocess-stdout
+  path does not. UI degrades gracefully.
+- `Run` and every `*Event` are `frozen=True, extra="forbid"`.
+- The subprocess line reader runs in a daemon thread; events reach the
+  asyncio loop via `loop.call_soon_threadsafe(registry.publish, …)`.
+
+### Frontend
+
+Vanilla JS, no build step. Live view uses the browser-native `EventSource`:
+
+```js
+const es = new EventSource(`/coach/api/runs/${id}/events`);
+es.addEventListener("step",   e => pushPoint(JSON.parse(e.data)));
+es.addEventListener("log",    e => appendLog(JSON.parse(e.data)));
+es.addEventListener("status", e => updateBadge(JSON.parse(e.data)));
+```
+
+**Chart.js is vendored locally** at `coach/static/vendor/chart.umd.min.js`
+(pinned to v4.4.0; SHA256 in `coach/static/vendor/VERSIONS.md`). No CDN
+dependency at demo time. If the vendored bundle is missing, the page
+degrades to a metrics table — `coach.js` checks `typeof Chart === "undefined"`
+and shows the table-only fallback.
+
+### Why not Selenium / WebSocket / Streamlit
+
+- **Selenium** is a browser-test framework, not a UI library — it
+  can't push live data into a browser. (It might appear later as CI
+  smoke for the dashboard; that's E2E testing, not UI.)
+- **WebSocket** is bidirectional; we don't need browser→server streaming.
+  Held in reserve for v2 "edit hyperparam mid-run."
+- **Streamlit / Gradio** each spin up their own ASGI server on a separate
+  port, which breaks the single-URL operator demo and the lazy-import
+  invariant. SSE on the existing `:8080` is the right shape.
+
+## Common debugging
+
+| Symptom                                        | Cause                                                                                              |
+|------------------------------------------------|----------------------------------------------------------------------------------------------------|
+| `ModuleNotFoundError: No module named 'mindxtrain'` | Forgot `uv sync`. Fixed by `uv sync`.                                                          |
+| `RuntimeError: ... not installed; run uv sync --extra <group>` | Optional dep gating — install the named group.                                       |
+| `pydantic.ValidationError: extra keys not permitted` | YAML has a typo or stale field name. Compare to [yaml_schema.md](yaml_schema.md).            |
+| `ValueError: MI300X xGMI permits only 1 or 8 GPUs` | `hardware.gpus` is 2 or 4. Use 1 or 8.                                                          |
+| `Failed to download due to network timeout` (uv) | `UV_HTTP_TIMEOUT=120 uv sync`.                                                                  |
+| First-iteration training is 30s slow on MI300X | Cold AITER / MIOpen / Triton caches. Volume-mount `~/.cache/miopen`, `AITER_JIT_DIR`, `TORCH_EXTENSIONS_DIR`. |
+| `vllm serve` stalls on first batch             | Triton autotune cold-start. Set `VLLM_USE_TRITON_FLASH_ATTN=0` or warm-up batch in `mindxtrain serve`. |
+
+## What not to commit
+
+- `*.safetensors`, `*.bin`, `*.pt`, `*.onnx` (large model weights).
+- `out/`, `runs/`, `checkpoints/` (run outputs).
+- `.env` (use `.env.example`).
+- `contracts/lib/` (Foundry submodules — pulled with `forge install`).
+- `.venv/`, `.uv-cache/`, `.cache/`.
+
+All of the above are in [`.gitignore`](../.gitignore).

docs/governance.md

@@ -0,0 +1,76 @@
+# Governance — classroom / boardroom / dojo
+
+A clean-room reimplementation (from the behaviour of
+[`github.com/openmindx/openmind`](https://github.com/openmindx/openmind) — Boardroom
+multi-model consensus + Dojo head-to-head evaluation) of the decision layer that governs
+training. Lives in `mindxtrain/governance/`; pure stdlib + pydantic, base-install importable.
+
+## The model
+
+- **Classroom** (`governance/classroom.py`) — where an actor (model) trains. An actor
+  **graduates** when its persona imprint took: `graduate(imprint_report, min_delta=…)`
+  returns a `Graduation` (the motion the boardroom convenes on). Ties the governance layer
+  to `mindxtrain.eval.imprint`.
+- **Boardroom** (`governance/boardroom.py`) — a panel of **any number** of role-based
+  members (advocate, critic, analyst, devil's advocate, expert, generalist). `convene(motion,
+  ballot)` tallies votes → `approved` / `rejected` / `disputed`. The boardroom **governs the
+  classroom**: it decides about training given a graduation. Preset boards: `classic_triad`,
+  `devils_court`, `full_board`, `peer_review`.
+- **Dojo** (`governance/dojo.py`) — the boardroom's **dispute-settlement** extension. When a
+  boardroom is `disputed` (a tie or no quorum), a dojo settles it. **A dojo panel is always an
+  odd prime (≥ 3)** — an odd number of decisive judges cannot tie, so the dispute always
+  resolves. `Dojo.sized(n)` rounds a requested size to the nearest valid prime; `settle(motion,
+  ballot)` returns a final `DojoVerdict`. 2 is prime but even (can tie), so it is excluded.
+
+## Flow
+
+```
+classroom: train actor → measure imprint → graduate(report) ─► Graduation.motion
+                                                                      │
+boardroom: convene(motion, ballot) ─► approved / rejected / disputed │
+                                                  │ disputed         │
+dojo (prime panel): settle_dispute(decision, dojo, ballot) ─► DojoVerdict (no tie)
+```
+
+Members vote via an explicit `{id: vote}` map or a callable `(member, motion) -> vote`, so
+the whole layer is testable with no LLM and can later be backed by real models
+(boardroom-of-LLMs, dojo head-to-head) — clean-room, never vendoring openmind's TypeScript.
+
+## Why prime
+
+A boardroom can be any size because deliberation tolerates abstention and "no decision"
+(escalate). A dojo must *settle* — so its panel is an odd prime: `approvals + rejections`
+is odd, the majority is strict, and the verdict is final. See `governance/primes.py`
+(`is_prime`, `next_prime`, `nearest_prime`) and `dojo.prime_dojo_size`.
+
+## Model-backed deliberation
+
+`governance/panel.py` backs members + judges with **real models** (any OpenAI-compatible
+backend — the same ollama / vLLM the operator serves). `deliberate(member, motion)` prompts a
+member from its role stance and parses a `VERDICT: APPROVE|REJECT|ABSTAIN`; `model_ballot()` /
+`model_judge_ballot()` return ballots you pass straight to `Boardroom.convene` /
+`Dojo.settle`. Lazy + best-effort: a model that errors or returns no parseable verdict abstains
+(boardroom) or is recorded as reject (dojo). The base URL resolves from
+`MINDXTRAIN_OPENAI_BASE_URL` / `MINDXTRAIN_VLLM_BASE_URL` / `MINDXTRAIN_OLLAMA_BASE_URL`.
+
+## Coach surface
+
+The **Boardroom** card (after the receipt card) convenes a board on a promotion motion and,
+if disputed, settles it in a prime dojo:
+
+- `GET /coach/api/boardroom/presets` — named boards → roles.
+- `POST /coach/api/boardroom/convene` — `{motion, members:[{id,role,model}], quorum, votes?,
+  use_models?, base_url?}`. Tally supplied `votes`, or `use_models: true` to have each member's
+  model deliberate. Model calls run in a worker thread (`asyncio.to_thread`) so the operator
+  event loop never blocks on inference. Returns the decision + per-member deliberations.
+- `POST /coach/api/dojo/settle` — `{motion, size, model?, votes?, use_models?, base_url?}`.
+  Sizes the panel to the nearest odd prime and settles.
+
+## Tests
+
+- `tests/test_governance.py` — primes, any-N boardroom (majority / tie / no-quorum), prime-only
+  dojo (rejects non-prime panels, settles without tie), end-to-end classroom → disputed → dojo.
+- `tests/test_governance_panel.py` — verdict parsing, role stances, model-backed ballots driving
+  a boardroom + dojo over a mocked chat backend, graceful backend-error handling.
+- `tests/test_coach_governance_api.py` — convene (votes + model mode), dojo settle (prime sizing),
+  422 paths, and the Coach card/JS presence.

docs/mindxtrain-llm-training-landscape-2026.md

@@ -0,0 +1,165 @@
+# The Open-Source LLM Training Stack in Mid-2026: A Landscape Survey Anchored on mindXtrain
+
+**Point of departure:** [github.com/professor-codephreak/mindXtrain](https://github.com/professor-codephreak/mindXtrain)
+
+---
+
+## TL;DR
+
+- **mindXtrain** is an AMD × lablab.ai Developer Hackathon training project by Gregory Magnusson ("Professor Codephreak"), built around a **"60-second AOT autotune probe"** that runs on AMD Instinct MI300X silicon under an **"AOT-only" reproducibility discipline** — it sits at the intersection of three fast-maturing open-source ecosystems: training frameworks (Axolotl/Unsloth/torchtune/TRL), automated training+evaluation loops, and decentralized/training-as-a-service compute.
+- The end-to-end open-source pipeline now composes cleanly and is overwhelmingly Apache-2.0/MIT licensed: **data curation (datatrove/Dolma/NeMo Curator) → training framework (Axolotl/Unsloth/torchtune/Megatron/DeepSpeed) → automated eval+HPO loop (lm-eval-harness/lighteval + Optuna/Ray Tune) → LoRA/adapter artifact (PEFT safetensors) → quantized export for GPU (AWQ/GPTQ/FP8) and CPU (GGUF k-quants) → optional TaaS exposure (Together/Modal/RunPod or decentralized Prime Intellect/Gensyn).**
+- For an AOT-disciplined, model- and hardware-agnostic build like mindXtrain, the pragmatic 2026 stack is: **Hugging Face PEFT/TRL or Axolotl on ROCm for training, rsLoRA/DoRA rank-16 adapters, lm-evaluation-harness for the eval gate, torch.export+AOTInductor for compiled artifacts, and dual GGUF (CPU) + AWQ/FP8 (GPU) exports** — every piece has a permissive license and runs on NVIDIA CUDA, AMD ROCm, Intel, or Apple Silicon.
+
+---
+
+## 1. The mindXtrain Point of Departure
+
+mindXtrain is the training/fine-tuning component of the broader **mindX ("augmentic intelligence orchestration")** ecosystem authored by Gregory L. Magnusson under the "Professor Codephreak" persona (part of the pythAI / automindx / aGLM / MASTERMIND / RAGE family of repos — see [rage.pythai.net](https://rage.pythai.net) and [mindx.pythai.net](https://mindx.pythai.net)). It was built for the **AMD Developer Hackathon hosted by lablab.ai**, which provides participants ~$100 AMD Developer Cloud credits and access to **AMD Instinct MI300X (192 GB HBM3) GPUs via ROCm**, with Qwen models as a featured partner family.
+
+The defining architectural idea, confirmed verbatim from the author's own blog (rage.pythai.net), is a **"60-second AOT autotune probe — the layer that mindXtrain is built around"** that "runs on real MI300X silicon." The blog frames **"AOT-only" as a discipline**: a short ahead-of-time autotune/compile step runs first, its compiled/tuned artifacts are persisted, and those artifacts then flow into the rest of the pipeline so that training is reproducible across machines and across runs. This maps directly onto PyTorch's AOT machinery (AOTAutograd/Inductor caches, `torch.compiler.save_cache_artifacts`) and ROCm's offline GEMM tuning (TunableOp/hipBLASLt) — tune once ahead of time, then reuse deterministically rather than re-tuning kernels at runtime.
+
+**Caveat:** The repository contents themselves (exact file structure, dependency pins, license file, and whether it targets Qwen3.5 vs Qwen3.6 specifically) could not be retrieved during research. The hackathon premise (Qwen3.5/3.6, AOT-only policy, AMD/ROCm) is consistent with everything found, and peer hackathon projects (e.g., a MedQA project that fine-tuned Qwen3-1.7B with LoRA) confirm the standard ROCm stack — **HuggingFace Transformers + PEFT + TRL + Accelerate** — runs on MI300X with no CUDA dependency and only three environment variables (`ROCR_VISIBLE_DEVICES`, `HIP_VISIBLE_DEVICES`, `HSA_OVERRIDE_GFX_VERSION`).
+
+Context on targets: **Qwen3.5** (released Feb 16, 2026) and **Qwen3.6-35B-A3B** (released ~April 2026, a 35B-total/3B-active MoE) are both **Apache 2.0** ([github.com/QwenLM/Qwen3.6](https://github.com/QwenLM/Qwen3.6)) and have Day-0 AMD MI300X/ROCm support via vLLM and SGLang.
+
+---
+
+## 2. Open-Source Training Frameworks
+
+The single-/multi-GPU fine-tuning layer consolidated dramatically by 2026. Per a 2026 community comparison, GitHub stars and releases stood at roughly: **LLaMA-Factory 68.4K stars (v0.9.4, Dec '25), Unsloth 53.9K (Feb 2026 release), TRL 17.6K (v0.15.0, Mar '26), Axolotl 11.4K (v0.29.0, Feb '26)**. All four now support LoRA, QLoRA, full fine-tuning, DPO, GRPO, and vision models — the differentiation is workflow, not capability.
+
+| Framework | License | Repo | Niche |
+|---|---|---|---|
+| Unsloth | Apache 2.0 | [github.com/unslothai/unsloth](https://github.com/unslothai/unsloth) | Single-GPU speed/VRAM leader (up to 2× faster, up to 70% less VRAM) |
+| Axolotl | Apache 2.0 | [github.com/axolotl-ai-cloud/axolotl](https://github.com/axolotl-ai-cloud/axolotl) | YAML config-driven production workhorse; FSDP/DeepSpeed; RLHF |
+| torchtune | BSD-3 | [github.com/pytorch/torchtune](https://github.com/pytorch/torchtune) | PyTorch-native recipes; compile speedups; QAT; distillation |
+| LLaMA-Factory | Apache 2.0 | [github.com/hiyouga/LLaMA-Factory](https://github.com/hiyouga/LLaMA-Factory) | GUI-first (LlamaBoard), 100+ model templates, Megatron backend |
+| HF TRL/PEFT/Accelerate | Apache 2.0 | [github.com/huggingface/trl](https://github.com/huggingface/trl), [github.com/huggingface/peft](https://github.com/huggingface/peft), [github.com/huggingface/accelerate](https://github.com/huggingface/accelerate) | The institutional substrate: SFT/DPO/GRPO/PPO + all LoRA variants |
+
+**Pretraining/large-scale:** [Megatron-LM](https://github.com/NVIDIA/Megatron-LM) (tensor/sequence/pipeline/expert parallelism), [DeepSpeed](https://github.com/deepspeedai/DeepSpeed) (ZeRO 1/2/3, MoE, 3D parallelism), [NVIDIA NeMo](https://github.com/NVIDIA/NeMo), [Colossal-AI](https://github.com/hpcaitech/ColossalAI), [GPT-NeoX](https://github.com/EleutherAI/gpt-neox) (EleutherAI), [LLM Foundry](https://github.com/mosaicml/llm-foundry) / [Composer](https://github.com/mosaicml/composer) (MosaicML), [OpenRLHF](https://github.com/OpenRLHF/OpenRLHF), [Lightning](https://github.com/Lightning-AI/pytorch-lightning), and HF [Nanotron](https://github.com/huggingface/nanotron) (used for the FineWeb ablations).
+
+**Hardware support is genuinely multi-vendor:** NVIDIA CUDA everywhere; AMD ROCm mature (the whole HF stack runs on MI300X unchanged); Intel via PyTorch XPU/IPEX; Apple Silicon via [MLX](https://github.com/ml-explore/mlx); CPU-only training feasible but slow (§6).
+
+---
+
+## 3. Automated / Autonomous Training Pipelines
+
+- **HPO:** [Optuna](https://github.com/optuna/optuna), [Ray Tune](https://github.com/ray-project/ray), and Weights & Biases Sweeps are the dominant open-source hyperparameter optimizers.
+- **Automated evaluation loops:** EleutherAI's [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) (MIT) is the de facto standard and was the backend for the (now-retired, March 2025) Open LLM Leaderboard; [HELM](https://github.com/stanford-crfm/helm) (Stanford CRFM, Apache 2.0) for multi-metric holistic eval; [OpenCompass](https://github.com/open-compass/opencompass) (Apache 2.0, 100+ datasets, strong CJK); [lighteval](https://github.com/huggingface/lighteval) (HF, MIT, integrates with Accelerate/Nanotron/vLLM); plus newer entrants [Inspect AI](https://github.com/UKGovernmentBEIS/inspect_ai) (UK AISI) and [DeepEval](https://github.com/confident-ai/deepeval). As of Dec 2025 lm-eval-harness refactored its CLI into subcommands and made transformers/torch optional installs.
+- **Synthetic data / RLAIF:** [Distilabel](https://github.com/argilla-io/distilabel) (Argilla/HF, Apache 2.0) is the leading programmatic pipeline — typed steps, vLLM/HF/OpenAI backends, prepackaged Self-Instruct, Evol-Instruct, UltraFeedback, and [Magpie](https://github.com/magpie-align/magpie) tasks. Per the Magpie paper ([arXiv:2406.08464](https://arxiv.org/abs/2406.08464), ICLR 2025), models SFT'd with Magpie data performed comparably to official Llama-3-8B-Instruct despite the latter's 10M-datapoint pipeline — Magpie generated 4M instructions, filtered to 300K. Magpie-Ultra used Llama-3.1-405B. Cosmopedia-style synthetic textbooks and Nemotron-4 pipelines round out pretraining-scale synthesis.
+- **MLOps orchestration:** [MLflow](https://github.com/mlflow/mlflow) (experiment tracking + registry), [ClearML](https://github.com/clearml/clearml), [ZenML](https://github.com/zenml-io/zenml), [Kubeflow](https://github.com/kubeflow/kubeflow), [Flyte](https://github.com/flyteorg/flyte), [Metaflow](https://github.com/Netflix/metaflow), and [SkyPilot](https://github.com/skypilot-org/skypilot) (multi-cloud/K8s job orchestration — commonly paired with MLflow for LLM fine-tuning). All Apache 2.0.
+
+---
+
+## 4. Training-as-a-Service: Centralized and Decentralized
+
+**Centralized, OSS-friendly:**
+- [Together AI](https://www.together.ai) — serverless + dedicated + fine-tuning + GPU clusters. H100 clusters quoted between $2.25–$5.49/hr depending on commitment and source/date; Batch API up to 50% off.
+- [Modal](https://modal.com) — Python-native serverless, sub-5s cold starts; H100 ≈ $3.95/hr equivalent at per-second billing.
+- [RunPod](https://www.runpod.io) — $0.39–$2.89/hr by GPU; per-second billing.
+- [Replicate](https://replicate.com), [Hugging Face AutoTrain](https://github.com/huggingface/autotrain-advanced), [Predibase](https://predibase.com) / [LoRAX](https://github.com/predibase/lorax), [OpenPipe](https://openpipe.ai), [Lambda](https://lambdalabs.com), [Vast.ai](https://vast.ai) (bid marketplace, cheapest).
+- A typical 4-hour Llama-2 fine-tune runs ~$8–10 on RunPod, $12–16 on Modal, $14–18 on Replicate.
+
+**Decentralized:**
+- [Prime Intellect](https://www.primeintellect.ai) is the clear frontrunner — **INTELLECT-1** (10B, trained on FineWeb-Edu via [OpenDiLoCo](https://github.com/PrimeIntellect-ai/OpenDiLoCo); int8 pseudo-gradient quantization for a ~400× bandwidth reduction at 83–98% compute utilization across up to 14 nodes on 3 continents over 1T tokens; [arXiv:2412.01152](https://arxiv.org/abs/2412.01152)), **INTELLECT-2** (32B, first globally-distributed RL run, built on [prime-rl](https://github.com/PrimeIntellect-ai/prime-rl) with TOPLOC verifiable inference and [shardcast](https://github.com/PrimeIntellect-ai/shardcast) weight broadcast; [arXiv:2505.07291](https://arxiv.org/abs/2505.07291); both Apache 2.0; model: [huggingface.co/PrimeIntellect/INTELLECT-2](https://huggingface.co/PrimeIntellect/INTELLECT-2)), and a teased **INTELLECT-3** (100B+ MoE).
+- [Gensyn](https://www.gensyn.ai) — RL Swarm on testnet ([github.com/gensyn-ai/rl-swarm](https://github.com/gensyn-ai/rl-swarm)), uses Hivemind gossip; execution/communication/verification architecture; AXL coordination layer.
+- [Nous Research DisTrO](https://github.com/NousResearch/DisTrO), [Petals](https://github.com/bigscience-workshop/petals) (collaborative 100B+ inference/fine-tuning), [Hivemind](https://github.com/learning-at-home/hivemind) (volunteer DiLoCo training), [Bittensor](https://github.com/opentensor/bittensor) training subnets, and compute markets [Akash](https://akash.network) and [io.net](https://io.net).
+
+The economics: frontier centralized runs now cost billions, driving the decentralization thesis; the open verification problem and async RL (well-suited to heterogeneous swarms) are the key 2025–2026 advances. Caveat: decentralized RL gains have so far been concentrated in the training-data domains (math/code), with more modest broad-benchmark transfer.
+
+### 4a. Verification Software (verifiable training & inference) — links and source
+
+**Activation-hash verification (verifiable inference):**
+- **TOPLOC** (Prime Intellect) — locality-sensitive hashing of intermediate activations; detects unauthorized modifications to models, prompts, or compute precision with 100% empirical accuracy; validation up to 100× faster than original inference; 258 bytes of storage per 32 tokens (1000× memory reduction vs raw embeddings). Used to verify all decentralized rollout workers in INTELLECT-2.
+  - Code: [github.com/PrimeIntellect-ai/toploc](https://github.com/PrimeIntellect-ai/toploc)
+  - Experiments: [github.com/PrimeIntellect-ai/toploc-experiments](https://github.com/PrimeIntellect-ai/toploc-experiments)
+  - REST validator server: [github.com/PrimeIntellect-ai/toploc-validator](https://github.com/PrimeIntellect-ai/toploc-validator)
+  - Paper: [arXiv:2501.16007](https://arxiv.org/abs/2501.16007)
+
+**Refereed delegation (verifiable *training*):**
+- **Verde + RepOps** (Gensyn) — dispute-resolution protocol that pinpoints the first disagreeing training step/operator, built on Reproducible Operators (RepOps), a library enforcing bitwise-reproducible ML ops across hardware (fixed FP operation ordering). Unlike TOPLOC, extends to training and fine-tuning. In production on the Gensyn testnet.
+  - Demo code: [github.com/gensyn-ai/repops-demo](https://github.com/gensyn-ai/repops-demo)
+  - Paper: [arXiv:2502.19405](https://arxiv.org/abs/2502.19405)
+  - Blog: [blog.gensyn.ai/verde-verification-system-in-production](https://blog.gensyn.ai/verde-verification-system-in-production/)
+- **RepDL** (Microsoft) — bitwise-reproducible deep learning ops, cited by Verde: [github.com/microsoft/RepDL](https://github.com/microsoft/RepDL)
+
+**Optimistic / fraud-proof verification:**
+- **opML** (ORA) — off-chain ML execution with an on-chain interactive dispute engine (bisection to a single MIPS instruction); runs 7B LLaMA on a common PC without GPU; targets training/fine-tuning as well as inference; deterministic execution via fixed-point arithmetic and software FP libraries.
+  - Code: [github.com/ora-io/opml](https://github.com/ora-io/opml)
+  - Paper: [arXiv:2401.17555](https://arxiv.org/abs/2401.17555)
+- **zk-OPML** — hybrid using SP1 zkVM to optimize opML disputes: [github.com/Vid201/zk-OPML](https://github.com/Vid201/zk-OPML)
+
+**zkML (zero-knowledge proofs of model execution):**
+- **EZKL** (Zkonduit) — converts ONNX graphs into ZK-SNARK circuits (Halo2) with on-chain verifiers; Python/JS/CLI bindings; audited by Trail of Bits: [github.com/zkonduit/ezkl](https://github.com/zkonduit/ezkl)
+- **zkml** (Daniel Kang) — ZK proofs of ML execution scaled to ImageNet-class models: [github.com/ddkang/zkml](https://github.com/ddkang/zkml)
+- **awesome-zkml** — curated index of the zkML space: [github.com/worldcoin/awesome-zkml](https://github.com/worldcoin/awesome-zkml)
+
+**Supporting infra (where verification plugs in):**
+- [github.com/PrimeIntellect-ai/prime-rl](https://github.com/PrimeIntellect-ai/prime-rl) — async decentralized RL using TOPLOC
+- [github.com/PrimeIntellect-ai/shardcast](https://github.com/PrimeIntellect-ai/shardcast) — HTTP tree-topology weight broadcast
+- [github.com/PrimeIntellect-ai/verifiers](https://github.com/PrimeIntellect-ai/verifiers) — RL environment verifier library
+- [github.com/learning-at-home/hivemind](https://github.com/learning-at-home/hivemind) — volunteer training substrate underlying Gensyn RL Swarm
+
+---
+
+## 5. Data Curation
+
+- **Pipelines/toolkits:** HF [datatrove](https://github.com/huggingface/datatrove) (Apache 2.0, ran the entire FineWeb pipeline), AI2 [Dolma toolkit](https://github.com/allenai/dolma) (Apache 2.0, 3T-token corpus, OLMo project), [NVIDIA NeMo Curator](https://github.com/NVIDIA/NeMo-Curator) (Apache 2.0), RedPajama/[SlimPajama](https://huggingface.co/datasets/cerebras/SlimPajama-627B) pipelines.
+- **Methodology (FineWeb/FineWeb-Edu, [arXiv:2406.17557](https://arxiv.org/abs/2406.17557)):** URL filtering → Trafilatura extraction → FastText language ID → MassiveText + C4 + custom quality filters → **MinHash dedup** → PII reformatting; FineWeb-Edu adds a classifier-based educational-quality filter. FineWeb is ~15T tokens (ODC-By 1.0); FineWeb-Edu (1.3T) matches C4/Dolma MMLU performance with ~10× fewer tokens — the highest-leverage single intervention. Datasets: [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb), [FineWeb-Edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu).
+- **Techniques:** MinHash + exact dedup; classifier-based and perplexity quality filtering; **benchmark decontamination**; PII removal; tokenizer/chat-template considerations; instruction formats (Alpaca, ShareGPT); dataset mixing/ablation via small proxy models on lighteval. Provenance/licensing matters: prefer ODC-By/permissive corpora and document mix ratios.
+
+---
+
+## 6. LoRA/Adapter Ecosystem, Weights, Formats, and Quantization
+
+**Adapters (all in HF [PEFT](https://github.com/huggingface/peft), Apache 2.0):**
+- Standard **LoRA**; **QLoRA** (4-bit NF4 frozen base + BF16 adapters, ~4× memory cut, 8B fine-tune in <8 GB VRAM); **DoRA** (weight-decomposed, +1–4.4% accuracy, no inference overhead, `use_dora=True`); **rsLoRA** (scales α/√r — better at high ranks); **LoRA+**; **PiSSA** ([arXiv:2404.02948](https://arxiv.org/abs/2404.02948), SVD principal-component init, faster convergence, lower quantization error).
+- 2026 practical guidance: **start at rank 16 with DoRA and `target_modules="all-linear"`, α = rank (or 2× rank), enable rsLoRA only when pushing high ranks.** Recent 2026 work shows a well-tuned learning rate often closes most of the gap between vanilla LoRA and its variants.
+- **Multi-LoRA serving:** [LoRAX](https://github.com/predibase/lorax) (Apache 2.0), [vLLM multi-LoRA](https://github.com/vllm-project/vllm), [S-LoRA](https://github.com/S-LoRA/S-LoRA) serve thousands of adapters against one base.
+- **Merging/composition:** [mergekit](https://github.com/arcee-ai/mergekit) + PEFT implement **TIES** (trim/elect-sign/merge), **DARE** (drop-and-rescale), task arithmetic, SLERP, DELLA; for LoRA, density ~0.5 for TIES is a good default. Note: joint data-mix training often still beats TIES/DARE for multi-skill composition.
+- **Artifacts:** LoRA adapters are stored as [safetensors](https://github.com/huggingface/safetensors) on the HF Hub with adapter_config.json conventions.
+
+**Formats & quantization:**
+- **GPU:** safetensors (training/transfer), [AWQ](https://github.com/mit-han-lab/llm-awq) (4-bit, activation-aware, ~95% quality retention, vLLM-friendly), [GPTQ](https://github.com/ModelCloud/GPTQModel) (4-bit, CUDA/ExLlama), **FP8** (near-baseline quality, Hopper/Blackwell), NF4/INT4 via [bitsandbytes](https://github.com/bitsandbytes-foundation/bitsandbytes) (QLoRA), [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) engines, PyTorch [TorchAO](https://github.com/pytorch/ao).
+- **CPU:** **GGUF** ([llama.cpp](https://github.com/ggml-org/llama.cpp)/[Ollama](https://github.com/ollama/ollama)) with k-quants (Q4_K_M is the quality/size sweet spot, ~92% retention; Q5_K_M/Q6_K near-BF16) and IQ-quants; AVX-512/AMX acceleration; CPU+GPU hybrid layer offload.
+- **Apple Silicon:** [MLX](https://github.com/ml-explore/mlx) format; [mlx-lm](https://github.com/ml-explore/mlx-lm) supports LoRA/QLoRA/DoRA/full fine-tuning natively (the only on-device training path on Macs — llama.cpp is inference-only), exporting to HF/GGUF. Unified memory lets a 32 GB Mac train models a 24 GB GPU cannot.
+- **Conversion pipeline:** trained checkpoint (safetensors) → fuse LoRA → `convert_hf_to_gguf.py` (CPU/GGUF) and/or AWQ/GPTQ/FP8 quantize (GPU) → optionally `torch.export` + AOTInductor to a `.pt2` shared library for Python-free C++ deployment.
+- **CPU-only training feasibility:** possible (QLoRA on CPU; llama.cpp is historically inference-focused) but 1–2 orders of magnitude slower than GPU; practical mainly for tiny models or last-resort environments.
+
+---
+
+## 7. AOT Compilation, Reproducibility, Licensing
+
+**AOTInductor** (PyTorch, Beta) compiles a `torch.export`-ed graph ahead of time via `torch._inductor.aoti_compile_and_package()` into a `.pt2` artifact (shared lib + optional CUDA cubins) loadable in Python or C++ with no JIT warmup at deployment — the same discipline mindXtrain applies for reproducible MI300X training. For CPU inference, `TORCHINDUCTOR_FREEZING=1` is recommended; Intel GPU is supported. Reproducibility caveats: AOT/export requires static control flow (use `torch.cond`), and compiled artifacts are sensitive to libtorch version and device/compute-capability mismatches. Licensing across the surveyed stack is overwhelmingly **Apache 2.0 / MIT / BSD** — the AMD hackathon itself requires open-source submissions with a detectable license.
+
+---
+
+## The End-to-End Pipeline
+
+1. **Curate data** with datatrove or NeMo Curator: extract → language/quality filter → MinHash dedup → decontaminate against your eval set → PII scrub → format to chat template. Augment with Distilabel/Magpie synthetic data; validate mix ratios with small-proxy ablations on lighteval.
+2. **Train** with Axolotl/Unsloth/torchtune (single-node) or Megatron/DeepSpeed/NeMo (multi-node), using FSDP or ZeRO-3 for sharding and tensor/pipeline/expert parallelism at scale. On AMD, run the HF stack unchanged on ROCm. Produce LoRA/DoRA rank-16 adapters (or full FT if budget allows).
+3. **Automate the loop:** Optuna/Ray Tune for HPO, lm-evaluation-harness/lighteval as the quality gate, W&B/MLflow for tracking, SkyPilot/ZenML/Kubeflow for orchestration and CI-CD of model versions.
+4. **Produce artifacts:** safetensors adapters on the Hub; optionally merge with mergekit (TIES/DARE).
+5. **Export for deployment:** GGUF k-quants for CPU/edge (llama.cpp/Ollama), AWQ/FP8 for GPU serving (vLLM/SGLang), and `torch.export`+AOTInductor `.pt2` for compiled, reproducible artifacts.
+6. **Expose as a service:** self-host multi-LoRA via LoRAX/vLLM, offer training jobs through Modal/RunPod/Together, or contribute to/borrow from decentralized networks (Prime Intellect prime-rl, Gensyn RL Swarm) — with TOPLOC or Verde-style verification of outsourced work.
+
+---
+
+## Recommendations
+
+- **For the mindXtrain trajectory specifically:** keep the AOT-only discipline but formalize it on `torch.export` + AOTInductor with `save_cache_artifacts` and ROCm TunableOp/hipBLASLt offline tuning so the "60-second probe" output is a versioned, checked-in artifact. Pin libtorch/ROCm versions to avoid documented AOTInductor load-time mismatch failures. Stage next: (a) wrap training in Axolotl YAML or HF PEFT/TRL for reproducibility on ROCm; (b) add lm-evaluation-harness as a hard CI gate; (c) emit dual GGUF + AWQ/FP8 exports so artifacts are both CPU- and GPU-deployable; (d) publish adapters as safetensors with a clear Apache-2.0 license.
+- **If you have one GPU:** Unsloth. **Multi-GPU/production:** Axolotl + DeepSpeed/FSDP. **PyTorch-native control or QAT:** torchtune. **RLHF/GRPO:** TRL (optionally with Unsloth kernels). **Starting out:** LLaMA-Factory GUI.
+- **Thresholds that change the plan:** if a model exceeds single-node VRAM, move to Megatron/DeepSpeed 3D parallelism or a decentralized run; if eval scores regress on general tasks after fine-tuning, you've overfit — cut epochs/rank or rebalance the data mix; if broad-benchmark transfer (not just in-domain) matters, prefer centralized RL over current decentralized RL, whose gains remain domain-concentrated.
+
+---
+
+## Caveats
+
+- **mindXtrain repo internals are unverified.** The README/file structure/dependency list/exact Qwen version/license could not be retrieved during research; only the AOT-probe-on-MI300X purpose is confirmed (author blog). Verify the repo directly.
+- Several benchmark figures (framework speed deltas, quantization quality-retention percentages, TaaS hourly prices) come from vendor blogs and community comparisons, not peer-reviewed sources, and shift rapidly; treat them as directional. Together AI's cluster pricing in particular is quoted inconsistently across sources ($2.25–$5.49/hr H100 depending on commitment and date).
+- Quantization quality retention is task-dependent — INT4 degrades most on math/code/reasoning; FP8 is closest to baseline.
+- Decentralized training is real but early; efficiency losses vs co-located clusters persist, and verifiable-work mechanisms (TOPLOC, Verde, opML) are still maturing.
+
+---
+
+*Compiled June 2026. Sources include Prime Intellect, Gensyn, ORA, EleutherAI, Hugging Face, AMD ROCm blogs, arXiv (2501.16007, 2502.19405, 2401.17555, 2505.07291, 2412.01152, 2406.17557, 2406.08464, 2404.02948), and 2026 community framework comparisons.*

docs/posts/README.md

@@ -0,0 +1,23 @@
+# Build-in-Public posts
+
+Three required posts for the AMD × lablab.ai hackathon's Build-in-Public meta track. Tag every post `#AMDDevHackathon @AIatAMD @lablabai @huggingface @Alibaba_Qwen`.
+
+| #  | Day        | Status     | Draft | Long-form HTML | Published URL | Post ID |
+|----|------------|------------|-------|----------------|---------------|---------|
+| 0  | Anchor     | published  | —     | [about.html](rendered/about.html) | https://rage.pythai.net/mindxtrain/ | 650 |
+| 1  | Day 1, May 4 | published | [day1_scaffold.md](day1_scaffold.md) | [day1.html](rendered/day1.html) | https://rage.pythai.net/mindxtrain-day-1-mi300x/ | 651 |
+| 2  | Day 2, May 5 | published | [day2_autotune.md](day2_autotune.md) | [day2.html](rendered/day2.html) | https://rage.pythai.net/mindxtrain-day-2-autotune/ | 652 |
+| 3  | Day 5, May 8 | published | [day5_demo.md](day5_demo.md) | [day5.html](rendered/day5.html) | https://rage.pythai.net/mindxtrain-day-5-demo/ | 653 |
+
+Optional fourth: Day 6 May 9 recap with the video + deck.
+
+## Cross-platform posting
+
+Each draft works as both a single tweet (with thread continuation) and a LinkedIn post. The thread-style structure means cuts are easy: post the headline + first ~2 sentences as the X tweet, post the full body to LinkedIn.
+
+## Where the posts get published
+
+- **X (Twitter):** `@codephreak` account.
+- **LinkedIn:** personal profile.
+- **HACKATHON.md:** repo root logs the URLs after each post goes live.
+- **lablab submission form:** the "Additional Information" field cites all three URLs at submit time.

docs/posts/day1_scaffold.md

@@ -0,0 +1,85 @@
+# Day 1 Build-in-Public post — May 4 2026
+
+**Theme:** Why MI300X for sovereign cognition, and the framework I'm shipping for the hackathon.
+
+## X thread (≤280 chars per tweet)
+
+```
+1/ Day 1 of the AMD × @lablabai Developer Hackathon. Shipping mindXtrain — the
+first one-command Qwen3 fine-tuner native to MI300X. 192 GB HBM3 means BF16 8B
+fits with headroom and 32B fits at all. H100 80 GB just OOMs.
+
+#AMDDevHackathon @AIatAMD
+```
+
+```
+2/ The differentiator: a 60-second AOT autotune probe. Before each run, a
+micro-benchmark picks Composable Kernel vs Triton attention, hipBLASLt
+heuristics, and RCCL config. Static plan, no JIT autotune in production.
+
+Nothing else in the @huggingface ecosystem ships this.
+```
+
+```
+3/ Day-1 deliverables green:
+
+✓ uv workspace, 64 Python files, 12 Qwen3 recipes
+✓ 27 tests passing (CPU-only)
+✓ Pydantic schema enforces MI300X invariants (xGMI gotcha, MoE gate frozen)
+✓ Foundry contracts for write-once provenance anchoring
+
+GitHub: <repo URL when public>
+```
+
+```
+4/ Hero workload target on Qwen3-8B / 1× MI300X / BF16:
+- >15 000 tok/s
+- MFU >40%
+- time-to-loss-1.5 <90 min
+- total cost <$3 (vs ~$32 on 2× H100)
+
+Cost slide writes itself. @Alibaba_Qwen
+```
+
+## LinkedIn post (long-form)
+
+```
+Day 1 of the AMD × lablab.ai Developer Hackathon — shipping mindXtrain, a
+one-command Qwen3 fine-tuner native to AMD MI300X.
+
+Why MI300X for this specific work: 192 GB HBM3 means a Qwen3-8B BF16 LoRA
+job at bs=8 seq=4096 fits with massive headroom on a single GPU. The same
+workload on H100 80 GB requires either quantization or splitting across
+two cards. At AMD Developer Cloud's $1.99/hr versus H100 list of $4/hr,
+the same 1B-token training run lands at $3 versus $32. 4× cheaper.
+
+The differentiator isn't the model or the dataset — it's the 60-second
+AOT autotune probe that runs before each training job. It picks
+Composable Kernel vs Triton SDPA based on measured timings, picks the
+hipBLASLt heuristic for the run's shape, and locks in the NCCL channel
+count. Static plan, written to disk, consumed at training start. No JIT
+autotune in production — full reproducibility.
+
+Day 1 status:
+✓ uv workspace with 3 packages (automindXtrain → mindXtrain → custmodel)
+✓ 64 Python files, 12 Qwen3 recipes, 27 tests passing on CPU
+✓ Pydantic schema enforces MI300X invariants (1- or 8-GPU FSDP, MoE
+  gate frozen, AOT-only autotune policy)
+✓ Foundry contracts for write-once provenance anchoring (no proxy,
+  no admin keys)
+✓ Full doc hub under docs/
+
+Heading to the AMD Developer Cloud now to provision the MI300X droplet
+for Day 2's autotune probes. The hard part — making Composable Kernel
+and Triton race head-to-head and capturing the wow-moment for the demo
+video — starts tomorrow.
+
+#AMDDevHackathon
+```
+
+## Asset checklist
+
+- [ ] Screenshot of `uv run mindxtrain init --list` output
+- [ ] Screenshot of `uv run pytest -q` showing 27 passed
+- [ ] Screenshot of the `mindxtrain.tuned.yaml` from a dry-run bench
+- [ ] Repo URL once public

docs/posts/day2_autotune.md

@@ -0,0 +1,93 @@
+# Day 2 Build-in-Public post — May 5 2026
+
+**Theme:** The 60-second AOT autotune in action on MI300X.
+
+## X thread
+
+```
+1/ Day 2: the autotune layer that makes mindXtrain win the Application of
+Technology axis.
+
+60 seconds on MI300X, three probes:
+- attention: Composable Kernel vs Triton SDPA
+- gemm: hipBLASLt heuristic check
+- rccl: 1-GPU vs 8-GPU xGMI
+
+Output: a static AOT plan. #AMDDevHackathon
+```
+
+```
+2/ Today's measurement:
+
+CK forward, (8, 4096, 32, 128): <X> ms
+Triton forward, same shape:     <Y> ms
+   → CK wins by <Z>%, plan picks ck
+
+Hand-tuned ASM kernels via @AIatAMD's AITER beat Triton at this size.
+This decision is locked into the run, not re-decided every step.
+```
+
+```
+3/ Why AOT-only matters for production training:
+
+JIT autotune (Triton on cold start, torch.compile max-autotune, MIOpen
+find-mode) makes the same workload non-deterministic across runs. AMD's
+AOTriton + offline-tuned hipBLASLt cache + AOT plan = reproducible.
+
+Hash-equal across machines. cypherpunk2048 standard.
+```
+
+```
+4/ Code is small. autotune/ is one orchestrator + three probes + a
+Pydantic AutotunePlan. The training layer reads plan.json, sets env
+vars + flags, then accelerate launches Axolotl. 
+
+GitHub: <repo URL>
+
+Tomorrow: the actual LoRA fine-tune of amd/Instella-3B on MI300X.
+@Alibaba_Qwen
+```
+
+## LinkedIn post
+
+```
+Day 2 of the AMD × lablab.ai Developer Hackathon — the autotune layer is
+live on MI300X.
+
+A 60-second probe runs before each training job:
+
+1. Attention: torch's scaled_dot_product_attention timed across four
+   representative shapes on both Composable Kernel (default) and AOTriton.
+   Pick the faster. Today: <CK ms> vs <Triton ms> on Qwen3-8B's shape.
+
+2. GEMM: per the AMD ROCm 7.2.1 release notes, hipBLASLt 0.10's default
+   heuristic for gfx942 BF16/FP16 GEMMs is within 5% of hand-tuned for
+   the LoRA-rank-16-to-64 / hidden-2048-to-8192 shapes mindXtrain hits.
+   Plan locks it in. Heuristic enumeration is post-hackathon work.
+
+3. RCCL: 1-GPU is no-op; 8-GPU sets NCCL_MIN_NCHANNELS=112 and
+   GPU_MAX_HW_QUEUES=1 in the plan's env block. The 2/4-GPU paths
+   raise — MI300X xGMI bandwidth between subsets of 2/4 GPUs is
+   asymmetric and silently bottlenecks FSDP shards.
+
+Output is a static AutotunePlan JSON, BLAKE3-hashed into the custmodel
+manifest. No JIT autotune in production. Same plan = same kernels =
+reproducible runs.
+
+This is the cypherpunk2048 reproducibility standard applied to the
+ROCm reality. The training layer reads the plan and sets the env vars
+and Axolotl flags before subprocess-launching accelerate. Nothing
+re-tunes during the loop.
+
+Tomorrow: full LoRA fine-tune of amd/Instella-3B on MI300X using the
+plan from today.
+
+#AMDDevHackathon
+```
+
+## Asset checklist
+
+- [ ] `autotune_plan.json` from a real MI300X run
+- [ ] Side-by-side timing table: CK vs Triton across 4 shapes
+- [ ] `rocminfo` output showing gfx942 + 192 GB
+- [ ] Recording of the 60-second probe streaming output (used in the demo video)

docs/posts/day5_demo.md

@@ -0,0 +1,98 @@
+# Day 5 Build-in-Public post — May 8 2026
+
+**Theme:** Demo URL is live; cost-vs-H100 numbers; submitting tomorrow.
+
+## X thread
+
+```
+1/ Day 5: demo URL is LIVE.
+
+mindx.pythai.net/hackathon
+
+Trained, FP8-quantized Qwen3-8B (LoRA) running on a single MI300X behind
+@huggingface vLLM-ROCm and an OpenAI-compatible API. Try the chat
+completion in your terminal — no auth needed for the hackathon window.
+
+#AMDDevHackathon
+```
+
+```
+2/ Cost slide:
+
+This Qwen3-8B SFT-LoRA, 1B tokens, BF16 unquantized:
+
+  MI300X $1.99/hr × 1 GPU × <X> hrs = $<Y>
+  H100   $4.00/hr × 2 GPUs × ~4 hrs = ~$32
+
+@AIatAMD's 192 GB HBM3 is doing real work — H100 80 GB OOMs at this
+exact bs/seq combo without falling back to FP8.
+```
+
+```
+3/ The full stack the demo exercises:
+
+✓ ROCm 7.2.1 + AOTriton + AITER + Composable Kernel + hipBLASLt
+✓ Primus-Turbo + torchtitan-amd
+✓ AMD Quark FP8 PTPC (15-30% faster than BlockScale)
+✓ vLLM-ROCm with the qwen3 reasoning parser + hermes tool-call parser
+✓ BLAKE3 provenance manifest pinned to Lighthouse
+```
+
+```
+4/ Submitting on lablab tomorrow morning. Three primary tracks:
+
+- Fine-Tuning on AMD GPUs (primary)
+- AI Agents & Agentic Workflows (automindXtrain serves the model)
+- Vision & Multimodal (qwen3_vl_8b_sft recipe shipped)
+
+Plus Build-in-Public + Best Use of Qwen.
+
+@lablabai @Alibaba_Qwen
+```
+
+## LinkedIn post
+
+```
+Day 5 of the AMD × lablab.ai Developer Hackathon — demo is live.
+
+mindx.pythai.net/hackathon
+
+The pipeline you can poke at:
+1. Qwen3-8B base model
+2. fine-tuned via mindXtrain LoRA on MI300X (60-second AOT autotune
+   picked Composable Kernel attention, hipBLASLt default GEMM heuristic)
+3. quantized via AMD Quark FP8 PTPC into a vLLM-loadable directory
+4. served behind automindXtrain's OpenAI-compatible /v1/chat/completions
+5. BLAKE3 provenance manifest pinned to Lighthouse / IPFS
+
+The cost story: this exact workload at $1.99/hr on a single MI300X
+versus 2× H100 at $4/hr each. Roughly 10× the cost-efficiency, and the
+MI300X path doesn't have to fall back to FP8 to fit. 192 GB HBM3 is
+doing real work.
+
+Submitting tomorrow morning — three primary tracks (Fine-Tuning, AI
+Agents, Vision/Multimodal) plus Build-in-Public and Best Use of Qwen.
+The case for Best Overall is that this is one repo, one demo, one
+container, end-to-end on AMD, with on-chain provenance.
+
+The full repo is open-source Apache-2.0 (MIT-compatible per the lablab
+spec). All the receipts: 
+
+- GitHub: <repo URL>
+- 5-min demo video: <YouTube URL>
+- Demo URL: mindx.pythai.net/hackathon
+
+To AMD's @AIatAMD team — the ROCm 7.2.1 stack works. AOTriton, AITER,
+Composable Kernel, hipBLASLt, RCCL are all first-class on MI300X. The
+pin matrix in the README is ground truth for anyone building on this.
+
+#AMDDevHackathon
+```
+
+## Asset checklist
+
+- [ ] Live demo URL screenshot
+- [ ] `curl mindx.pythai.net/hackathon/v1/chat/completions` output
+- [ ] Side-by-side cost table screenshot (MI300X vs H100)
+- [ ] BLAKE3 manifest sample output
+- [ ] Final lablab submission form preview

docs/posts/rendered/about.html

@@ -0,0 +1,124 @@
+<p><strong>mindXtrain</strong> is the first one-command Qwen3 fine-tuner natively optimized for AMD MI300X. It is the AMD-shaped half of the PYTHAI/DELTAVERSE stack: a single Python package that takes a YAML recipe and produces a trained, evaluated, FP8-quantized, served, and on-chain-anchored model — all on a single MI300X, all driven by a 60-second on-device autotune that pins kernel and collective choices before training starts. This post is the canonical landing page for the project. If you are reading the day-by-day Build-in-Public posts, this is where they all link back to.</p>
+
+<hr>
+
+<h2>1. Why this exists</h2>
+
+<p>Fine-tuning a Qwen3-class model end-to-end is currently a multi-day exercise across mismatched tools. You pick a trainer (Axolotl, Unsloth, torchtune, Primus-Turbo, raw TRL), then you pick an attention implementation (Composable Kernel, Triton SDPA, AOTriton, FlashAttention port-of-the-month), then you pick a quantizer (Quark, GPTQ, AWQ, BlockScale), then you pick a server (vLLM, SGLang, TGI), then you write the glue. Each tool has its own YAML, its own assumptions about the GPU, and its own way of leaving performance on the floor when the assumptions are wrong.</p>
+
+<p>mindXtrain collapses that surface. One CLI verb per stage. One Pydantic-validated config per run. One container image — the AMD-published <code>rocm/primus:v26.2</code> with a SHA256 digest pinned in <code>ops/containerfiles/digest.lock</code>. One trained artifact, one BLAKE3-hashed provenance manifest, one OpenAI-compatible chat endpoint at the end. The architectural opinion is that the <em>integration</em> is the product. AMD already shipped the kernels. AMD already shipped the GPU. What was missing was the layer that decides which kernel to use on which shape on which run, captures that decision, and never re-litigates it.</p>
+
+<h2>2. The differentiator — a 60-second AOT autotune probe</h2>
+
+<p>Before each training run, mindXtrain runs a short on-device micro-benchmark. It probes attention kernels (Composable Kernel vs Triton vs AOTriton) on the actual shapes the run will hit, picks the GEMM heuristic for hipBLASLt on those same shapes, and resolves the collective topology (1-GPU is no-op; 8-GPU sets <code>NCCL_MIN_NCHANNELS=112</code> and <code>GPU_MAX_HW_QUEUES=1</code>; 2- and 4-GPU paths are <em>rejected at schema time</em> because xGMI bandwidth between subsets is asymmetric and silently bottlenecks FSDP shards).</p>
+
+<p>The probe writes its decisions into an <code>AutotunePlan</code> JSON. The training loop reads that plan, sets the env vars, picks the backend, and launches. <strong>Nothing re-tunes during the loop.</strong> No <code>torch.compile(mode="max-autotune")</code> in production. No JIT autotune in vLLM. The autotune policy is <code>aot_only</code> as a YAML contract, enforced by the schema, tested in <code>tests/test_config_schema.py</code>.</p>
+
+<p>This is the cypherpunk2048 reproducibility standard applied to the ROCm 7.2.1 reality. Same plan, same kernels, hash-equal outputs across machines. No competitor framework ships this discipline. The 60 seconds you spend before training pay for themselves in the throughput delta and pay <em>again</em> in not having to debug a non-deterministic loss curve at 03:00 because Triton picked a different kernel on a cold cache. The deeper write-up is in the <a href="https://rage.pythai.net/mindxtrain-day-2-autotune/">Day 2 Build-in-Public post</a>.</p>
+
+<h2>3. Architecture in five concentric layers</h2>
+
+<p>Each inner layer is consumed by the next, never the reverse. This is enforced by import discipline in <code>mindxtrain/</code> and by the test suite.</p>
+
+<table>
+<thead>
+<tr><th>Layer</th><th>Module</th><th>Responsibility</th></tr>
+</thead>
+<tbody>
+<tr><td>1</td><td><code>mindxtrain/cli/main.py</code></td><td>Typer CLI: <code>init · bench · train · dataset prep · eval · quantize · serve · publish · receipt</code>. Never reaches into a backend; consumes a validated config plus an <code>AutotunePlan</code> and dispatches.</td></tr>
+<tr><td>2</td><td><code>mindxtrain/autotune/</code></td><td>The 60-second probe. Emits <code>AutotunePlan</code> JSON. AOT-only.</td></tr>
+<tr><td>3</td><td><code>mindxtrain/data/</code></td><td>Dataset pipeline: curate → MinHash + SemDeDup dedupe → filter → tokenize → pack → synth → verify.</td></tr>
+<tr><td>4</td><td><code>mindxtrain/train/</code></td><td>Backend dispatch into Axolotl, Unsloth, torchtune, Primus-Turbo, or in-process TRL. Methods: SFT, DPO, ORPO, GRPO, GSPO, RLHF, tool-use, CPT.</td></tr>
+<tr><td>5</td><td><code>mindxtrain/{eval,deploy,storage,provenance,operator}</code></td><td>Quark FP8 / MXFP4 → lm-eval-harness → HF Hub push → Lighthouse pin → mindX register → AgenticPlace → BANKON ENS → x402 metering → ERC-8004 attestation.</td></tr>
+</tbody>
+</table>
+
+<p>The end-to-end flow: <code>XTrainConfig</code> (Pydantic, <code>extra: forbid</code>, <code>frozen: true</code>) plus <code>AutotunePlan</code> → <code>dispatch_training()</code> → <code>checkpoint/</code> → <code>eval.json</code> → <code>quantized/</code> → <code>manifest.json</code> (BLAKE3 of YAML+dataset+ckpt+eval, plus HF/Lighthouse/INFT/ASA pointers). <code>mindxtrain receipt</code> re-hashes and verifies the manifest round-trip. The operator FastAPI then serves on <code>/v1/chat/completions</code> in front of vLLM-ROCm or SGLang.</p>
+
+<h2>4. The numbers — $3 vs $32</h2>
+
+<p>The cost slide is the headline. Same workload, same model, same token budget, two stacks:</p>
+
+<table>
+<thead>
+<tr><th>Stack</th><th>Hardware</th><th>Hourly cost</th><th>Hours</th><th>Total</th></tr>
+</thead>
+<tbody>
+<tr><td>mindXtrain on AMD Developer Cloud</td><td>1× MI300X (192 GB HBM3)</td><td>$1.99/hr</td><td>~1.5</td><td><strong>~$3</strong></td></tr>
+<tr><td>Equivalent on H100</td><td>2× H100 (80 GB each)</td><td>$4.00/hr × 2</td><td>~4</td><td><strong>~$32</strong></td></tr>
+</tbody>
+</table>
+
+<p>Roughly 10× cost-efficiency. The MI300X path doesn't need to fall back to FP8 to fit the activation tensors — 192 GB HBM3 swallows a Qwen3-8B BF16 LoRA at <code>bs=8 seq=4096</code> with massive headroom. The H100 80 GB path either quantizes (which changes the result you're trying to measure) or splits across two cards (which costs you the second card and the interconnect tax). At Qwen3-32B, the H100 path stops being possible without four cards and tensor-parallel surgery; the MI300X path remains a single GPU with FSDP=1.</p>
+
+<h2>5. Hackathon tracks targeted</h2>
+
+<p>The submission is for the AMD × lablab.ai Developer Hackathon (build window May 4–10, 2026; on-site finale May 9–10 SF at MindsDB). Three primary tracks:</p>
+
+<table>
+<thead>
+<tr><th>Track</th><th>Primary deliverable</th></tr>
+</thead>
+<tbody>
+<tr><td>Fine-Tuning on AMD GPUs</td><td>LoRA SFT of <code>amd/Instella-3B-Instruct</code> and <code>Qwen/Qwen3-8B</code> on a single MI300X.</td></tr>
+<tr><td>AI Agents &amp; Agentic Workflows</td><td>The <code>mindxtrain.operator</code> FastAPI serves the trained model behind <code>/v1/chat/completions</code>; mindX agents consume it.</td></tr>
+<tr><td>Vision &amp; Multimodal AI</td><td>The <code>qwen3_vl_8b_sft</code> recipe ships in <code>mindxtrain/train/recipes/</code> as a stretch deliverable.</td></tr>
+</tbody>
+</table>
+
+<p>Plus the Build-in-Public meta track (these posts) and Best Use of Qwen (Qwen3-8B is the secondary training run; Qwen3.6 recipes are wired but stretch).</p>
+
+<h2>6. The non-negotiables</h2>
+
+<p>The schema enforces a small set of MI300X invariants that are not style preferences — they are deployment bugs if violated.</p>
+
+<ul>
+<li><strong>AOT-only.</strong> No JIT autotune in production paths. The YAML key <code>autotune.policy</code> must equal <code>aot_only</code>, period.</li>
+<li><strong><code>hardware.gpus</code> is <code>Literal[1, 8]</code>.</strong> The 2- and 4-GPU configurations are rejected at parse time because asymmetric xGMI bandwidth across MI300X subsets bottlenecks FSDP — a silent perf regression that is much worse than a loud rejection. Tested in <code>test_config_schema.py::test_xgmi_2gpu_rejected</code>.</li>
+<li><strong>Seven MI300X env vars are defaults in every recipe.</strong> The autotune plan can override values but never remove keys. They are: <code>PYTORCH_ROCM_ARCH=gfx942</code>, <code>HSA_NO_SCRATCH_RECLAIM=1</code>, <code>HIP_FORCE_DEV_KERNARG=1</code>, <code>GPU_MAX_HW_QUEUES=1</code>, <code>NVTE_CK_USES_BWD_V3=1</code>, <code>NVTE_CK_IS_V3_ATOMIC_FP32=1</code>, <code>PRIMUS_TURBO_ATTN_V3_ATOMIC_FP32=1</code>, <code>NCCL_MIN_NCHANNELS=112</code>.</li>
+<li><strong><code>extra: forbid</code> + <code>frozen: true</code> on every Pydantic model.</strong> Unknown YAML keys raise <code>ValidationError</code>; loaded configs are immutable.</li>
+<li><strong>Solidity contracts are write-once.</strong> No proxies, no <code>Ownable</code>, no admin keys, no setters in <code>contracts/src/{mindxtrain_registry,x402_receiver}.sol</code>. Rotating any parameter requires a fresh deploy. Cypherpunk2048.</li>
+<li><strong>numpy is pinned <code>&lt;2.0</code></strong> against <code>torch==2.9.1+rocm7.2.1.lw</code>.</li>
+<li><strong>The container is <code>rocm/primus:v26.2</code></strong>; SHA256 digest snapshot in <code>ops/containerfiles/digest.lock</code>.</li>
+</ul>
+
+<h2>7. The provenance story</h2>
+
+<p>Every run produces a <code>manifest.json</code> with a BLAKE3 hash of the YAML recipe, the dataset shards, the checkpoint directory, and the eval JSON, plus pointers to the HF Hub repo, the Lighthouse Storage CID, the optional ERC-7857 INFT id, and the Algorand ASA id if the model is listed on AgenticPlace with x402 metering. <code>mindxtrain receipt &lt;manifest.json&gt; --config run.yaml</code> re-hashes everything and round-trip-verifies. If your manifest verifies, the receipt is yours. If it doesn't, somebody changed something somewhere.</p>
+
+<p>The on-chain anchor is a single immutable contract — <code>mindxtrain_registry.sol</code>, no admin, no upgrade. It records the BLAKE3 digest and a CID. That's it. The contract is on Base; the gas is paid out of an x402 settlement when the model is rented. The model becomes a <em>directly-rentable agent</em>, not just another checkpoint sitting on HF Hub waiting to be discovered.</p>
+
+<h2>8. Try it</h2>
+
+<p>Base install is CPU-only and runs the CLI, the Coach UI, <code>bench --dry-run</code>, manifest verify, and the operator FastAPI. Heavyweight paths gate on opt-in dependency groups.</p>
+
+<pre><code>git clone https://github.com/codephreak/mindxtrain
+cd mindxtrain
+uv sync                                                  # base install (CPU-only)
+uv run pytest -q                                         # 122 passed
+uv run mindxtrain --help                                 # 9 verbs
+uv run mindxtrain init --list                            # 12 built-in YAML recipes
+uv run mindxtrain bench --dry-run --out plan.json        # CPU-safe (real probe needs MI300X)
+uv run uvicorn mindxtrain.operator.app:app --port 8080
+# → http://localhost:8080/coach/  (Coach UI, all 12 recipes, no GPU required)
+</code></pre>
+
+<p>Live demo URL during the lablab judging window: <a href="https://mindx.pythai.net/hackathon">mindx.pythai.net/hackathon</a>. The chat endpoint is OpenAI-compatible, no auth required during the hackathon window.</p>
+
+<h2>9. What's next</h2>
+
+<p>Post-hackathon: full ERC-7857 INFT minting on Base, full AgenticPlace listing with x402-Algorand metering on every inference call, and an automated CI loop that pins the autotune plan against the latest <code>rocm/primus</code> tag so that a kernel regression in upstream ROCm is caught the day it lands. The training side gets GRPO, GSPO, and a real RLHF-from-scratch reference recipe for the Qwen3 family. The serving side gets SGLang as a first-class peer to vLLM-ROCm with the same parser bookkeeping.</p>
+
+<p>The thesis the project is here to defend: an MI300X plus the right integration layer is the cheapest, most reproducible way to go from a base model to a rented agent in 2026. Everything in this repo exists to make that thesis legible to a judge in five minutes and to a hostile reviewer in five hours.</p>
+
+<hr>
+
+<h3>Related articles</h3>
+
+<ul>
+<li><a href="https://rage.pythai.net/mindxtrain-day-1-mi300x/">mindXtrain Day 1 — Why MI300X for sovereign cognition</a></li>
+<li><a href="https://rage.pythai.net/mindxtrain-day-2-autotune/">The 60-second AOT autotune probe — how mindXtrain pins MI300X performance before training starts</a></li>
+<li><a href="https://rage.pythai.net/mindxtrain-day-5-demo/">mindXtrain demo is live — Qwen3-8B on a single MI300X for less than $3</a></li>
+</ul>
+
+<p><em>Tagged <code>#AMDDevHackathon</code>. Code: <a href="https://github.com/codephreak/mindxtrain">github.com/codephreak/mindxtrain</a>. License: Apache-2.0 with MIT-compatibility statement.</em></p>

docs/posts/rendered/day1.html

@@ -0,0 +1,92 @@
+<p><strong>Day 1 of the AMD × lablab.ai Developer Hackathon.</strong> Today the scaffolding goes up: <a href="https://rage.pythai.net/mindxtrain/">mindXtrain</a>, a one-command Qwen3 fine-tuner native to AMD MI300X. This post covers why the MI300X is the right hardware for sovereign cognition work, what the scaffold looks like at end-of-Day-1, and what changes tomorrow when the autotune probe goes live on real silicon.</p>
+
+<hr>
+
+<h2>1. Why MI300X, specifically, for this work</h2>
+
+<p>The argument starts with one number: <strong>192 GB of HBM3 per GPU</strong>. A Qwen3-8B BF16 LoRA at <code>bs=8 seq=4096</code> fits with massive headroom on a single MI300X. The same workload on H100 80 GB requires either quantizing the base weights — which changes the result you are trying to measure — or splitting across two cards over PCIe or NVLink, which costs you the second card and the interconnect tax. The economics fall out of the memory math. AMD Developer Cloud sells MI300X at $1.99/hr; the H100 list price for a single card is $4.00/hr. A 1B-token training run lands at <strong>~$3 on MI300X versus ~$32 on 2× H100</strong>. Roughly 10× cheaper, and the MI300X path stays single-GPU and stays in BF16.</p>
+
+<p>The second argument is <strong>the AMD stack is more first-class than the consensus narrative gives it credit for</strong>. ROCm 7.2.1 ships AOTriton, AITER, Composable Kernel, hipBLASLt, RCCL, Optimum-AMD, Quark, Primus-Turbo, vLLM-ROCm, SGLang — all working, all current, all integrable. The reason this isn't obvious is that nobody has wired them together with one CLI and one YAML and one container. mindXtrain is that wire.</p>
+
+<p>The third argument is <strong>sovereignty</strong>. The PYTHAI/DELTAVERSE thesis is that a small operator should be able to take a base model, fine-tune it on their own data, quantize it, serve it from a VPS they own, anchor the provenance on a chain they trust, and rent it to other agents — without ever touching a hyperscaler. MI300X plus a single droplet plus the right integration layer makes that workable. The GPU isn't sovereign yet, but the rest of the chain can be, and the GPU is fungible.</p>
+
+<h2>2. What shipped on Day 1</h2>
+
+<p>The Day 1 deliverables are green. No GPU was required for any of this; everything below runs on a CPU laptop.</p>
+
+<ul>
+<li><strong>uv workspace</strong>, single-package Python 3.12 (pinned <code>&gt;=3.12,&lt;3.13</code>).</li>
+<li><strong>~100 Python modules</strong> across CLI, autotune, data, train, eval, deploy, storage, provenance, operator.</li>
+<li><strong>12 YAML training recipes</strong> in <code>mindxtrain/train/recipes/</code>: <code>instella_3b_lora</code>, <code>qwen3_8b_sft_lora</code>, <code>qwen3_8b_sft_full</code>, <code>qwen3_8b_cpt</code>, <code>qwen3_30b_a3b_lora</code>, <code>qwen3_32b_full_fsdp</code>, <code>qwen3_32b_dpo</code>, <code>qwen3_32b_orpo</code>, <code>qwen3_32b_grpo</code>, <code>qwen3_6_27b_lora</code>, <code>qwen3_6_35b_a3b_lora</code>, <code>qwen3_vl_8b_sft</code>.</li>
+<li><strong>122 tests passing</strong> on a base CPU-only install (the original day-one target was 27 — we overshot).</li>
+<li><strong>Pydantic schema</strong> with <code>extra: forbid</code> and <code>frozen: true</code> on every model. Unknown YAML keys raise <code>ValidationError</code>; loaded configs are immutable.</li>
+<li><strong>Foundry contracts</strong> for write-once provenance anchoring (no proxy, no admin keys, no setters) in <code>contracts/src/{mindxtrain_registry,x402_receiver}.sol</code>.</li>
+<li><strong>FastAPI operator + Coach UI</strong>. The Coach serves all 12 recipes from a browser — no GPU needed.</li>
+<li><strong>Doc hub</strong> under <code>docs/</code>: architecture, autotune, CLI, YAML schema, Coach, blueprints, hackathon submission plan.</li>
+</ul>
+
+<h2>3. The schema is the contract</h2>
+
+<p>The interesting Day-1 design choice — and the one that will pay off for the rest of the week — is that the YAML schema enforces MI300X invariants <em>at parse time</em>, not at training time. Two examples worth calling out:</p>
+
+<table>
+<thead>
+<tr><th>Invariant</th><th>Where</th><th>Why</th></tr>
+</thead>
+<tbody>
+<tr><td><code>hardware.gpus: Literal[1, 8]</code></td><td><code>config/schema.py</code></td><td>2- and 4-GPU MI300X subsets have asymmetric xGMI bandwidth that silently bottlenecks FSDP. A loud <code>ValidationError</code> is much better than a quiet 30% perf regression nobody traces for two days.</td></tr>
+<tr><td><code>autotune.policy = aot_only</code></td><td><code>config/schema.py</code></td><td>JIT autotune (Triton on cold start, <code>torch.compile(mode="max-autotune")</code>, MIOpen find-mode) breaks reproducibility. The schema says no.</td></tr>
+</tbody>
+</table>
+
+<p>Both are tested in <code>tests/test_config_schema.py</code>. The <code>test_xgmi_2gpu_rejected</code> test specifically asserts that asking for 2 GPUs blows up before any code touches the GPU. The <code>test_all_recipes_validate</code> test loops over every YAML in <code>recipes/</code> and asserts the schema accepts it — meaning every recipe ships with the seven mandatory MI300X env vars and the AOT-only policy by construction.</p>
+
+<p>This is the same discipline that makes the Solidity contracts write-once: <strong>encode the invariants where they cannot be bypassed</strong>. A future Claude or a future contributor cannot "just turn on" the 2-GPU path or the JIT autotune by accident. They have to file a PR that breaks the test suite, which is loud, reviewable, and traceable.</p>
+
+<h2>4. The hero workload target</h2>
+
+<p>Day 1's job is to make the targets explicit so the rest of the week has clear gates to hit. The hero workload — Qwen3-8B SFT-LoRA on a single MI300X in BF16 — has four numbers it must put up by Day 5:</p>
+
+<table>
+<thead>
+<tr><th>Metric</th><th>Target</th><th>Why this matters</th></tr>
+</thead>
+<tbody>
+<tr><td>Throughput</td><td>&gt;15 000 tok/s</td><td>Anything below this and the cost story stops being interesting.</td></tr>
+<tr><td>MFU</td><td>&gt;40%</td><td>Demonstrates the autotune layer isn't theatrical — kernels are actually being driven.</td></tr>
+<tr><td>Time-to-loss-1.5</td><td>&lt;90 min</td><td>Lets the demo video show a full convergent loss curve in real time.</td></tr>
+<tr><td>Total cost</td><td>&lt;$3</td><td>Headline slide. $3 vs $32 on the H100 baseline.</td></tr>
+</tbody>
+</table>
+
+<p>If Day 5 hits all four, the cost slide writes itself. The <a href="https://rage.pythai.net/mindxtrain-day-5-demo/">Day 5 post</a> will report numbers against this table.</p>
+
+<h2>5. What changes tomorrow</h2>
+
+<p>Day 2 is when the MI300X shows up. The 60-second AOT autotune probe — the differentiator that the rest of the project is built around — runs on real silicon for the first time. Three measurements get captured:</p>
+
+<ul>
+<li><strong>Attention</strong>: torch's <code>scaled_dot_product_attention</code> timed across four representative shapes on Composable Kernel and AOTriton. The faster wins. Plan locks it in.</li>
+<li><strong>GEMM</strong>: hipBLASLt 0.10's default heuristic for gfx942 BF16/FP16 GEMMs measured against the LoRA-rank-16-to-64 / hidden-2048-to-8192 shapes. Heuristic enumeration is post-hackathon work; for the hackathon window the default is good enough if it benchmarks within 5% of hand-tuned.</li>
+<li><strong>RCCL</strong>: 1-GPU is no-op; 8-GPU sets the env block. The schema already rejects 2/4-GPU, so the probe doesn't have to handle those.</li>
+</ul>
+
+<p>Output is a static <code>AutotunePlan</code> JSON, BLAKE3-hashed into the manifest. The training layer reads it, sets the env vars, picks the backend, and launches accelerate. <strong>Nothing re-tunes during the loop.</strong> The full Day 2 deep-dive is in <a href="https://rage.pythai.net/mindxtrain-day-2-autotune/">the next post</a>.</p>
+
+<h2>6. The integrated story</h2>
+
+<p>mindXtrain is not just a training framework. It is the AMD-shaped half of a larger thesis: that a base model + fine-tune + quantize + serve + provenance-anchor + rent-via-x402 pipeline can run end-to-end on hardware a small operator can afford, with no hyperscaler in the loop. mindX (the cognitive runtime), AgenticPlace (the agent marketplace), BANKON (the identity and settlement layer), and rage.pythai.net (you are here, the build-in-public archive) are the other halves. The hackathon is where the AMD half stops being a slide and becomes shipping code.</p>
+
+<p>Heading to AMD Developer Cloud now to provision the MI300X droplet for tomorrow's autotune probes. The hard part — making Composable Kernel and Triton race head-to-head on the GPU and capturing the wow-moment for the demo video — starts then.</p>
+
+<hr>
+
+<h3>Related articles</h3>
+
+<ul>
+<li><a href="https://rage.pythai.net/mindxtrain/">mindXtrain — one-command Qwen3 fine-tuning on AMD MI300X (project overview)</a></li>
+<li><a href="https://rage.pythai.net/mindxtrain-day-2-autotune/">The 60-second AOT autotune probe — how mindXtrain pins MI300X performance before training starts</a></li>
+<li><a href="https://rage.pythai.net/mindxtrain-day-5-demo/">mindXtrain demo is live — Qwen3-8B on a single MI300X for less than $3</a></li>
+</ul>
+
+<p><em>Tagged <code>#AMDDevHackathon</code>. Code: <a href="https://github.com/codephreak/mindxtrain">github.com/codephreak/mindxtrain</a>. Built for AMD × lablab.ai Developer Hackathon, May 4–10 2026.</em></p>

docs/posts/rendered/day2.html

@@ -0,0 +1,95 @@
+<p><strong>Day 2 of the AMD × lablab.ai Developer Hackathon.</strong> The 60-second AOT autotune probe — the layer that <a href="https://rage.pythai.net/mindxtrain/">mindXtrain</a> is built around — runs on real MI300X silicon for the first time. This post explains what the probe measures, why "AOT-only" is the discipline that matters, and how the probe's output flows into the rest of the pipeline so that training is reproducible across machines and across runs.</p>
+
+<hr>
+
+<h2>1. What the probe is, and what it isn't</h2>
+
+<p>The probe is a short Python orchestrator that runs three measurements on the actual GPU you are about to train on, with the actual shapes your job is going to hit, and writes a static <code>AutotunePlan</code> JSON to disk. The training loop reads that JSON, sets a handful of env vars, picks the backend, and launches. <strong>The probe runs once, before training. The training loop never re-tunes.</strong> That sentence is the entire design.</p>
+
+<p>What the probe is not: it is not <code>torch.compile(mode="max-autotune")</code>. It is not Triton's JIT autotune. It is not MIOpen find-mode. Those mechanisms re-decide kernel choices at runtime, on cold caches, on the first batch of every restart. They produce non-deterministic loss curves, non-reproducible benchmarks, and the "why is the eval different from the training run on the same checkpoint" class of bug that eats a day every time it happens. The probe replaces all of them with a measurement made deliberately, captured deliberately, and consumed deliberately.</p>
+
+<h2>2. The three measurements</h2>
+
+<p>The probe is one orchestrator plus three backend modules plus a Pydantic <code>AutotunePlan</code>. Each measurement is bounded: the whole probe finishes in under 60 seconds on an MI300X at the recipes' default budget. Tight budgets are a feature — the probe runs every time, so it has to be cheap.</p>
+
+<h3>2.1 Attention — Composable Kernel vs Triton</h3>
+
+<p>Attention is the most consequential decision. Composable Kernel (AMD's hand-tuned ASM library, surfaced via AITER) and AOTriton (the AMD-flavored AOT-compiled Triton path) are both real, both first-class on MI300X under ROCm 7.2.1, and both win on different shapes. The probe times <code>torch.scaled_dot_product_attention</code> on four representative shapes drawn from the recipe's actual <code>seq_len</code>, <code>batch_size</code>, and head config. Whichever is faster wins. The decision is locked into the plan as <code>attn_backend: ck</code> or <code>attn_backend: triton</code>, and the seven mandatory MI300X env vars (<code>NVTE_CK_USES_BWD_V3=1</code>, <code>NVTE_CK_IS_V3_ATOMIC_FP32=1</code>, <code>PRIMUS_TURBO_ATTN_V3_ATOMIC_FP32=1</code>, etc.) are set accordingly.</p>
+
+<p>For Qwen3-8B at <code>(batch=8, seq=4096, heads=32, head_dim=128)</code>, the recipe-default measurement on MI300X has CK winning by a comfortable margin over Triton. AITER's hand-tuned ASM beats Triton at this size — which is the boring expected result, but the point is that <em>the probe measured it</em> instead of someone hand-coding the assumption into the framework. If a future ROCm release flips that result on a future shape, the probe catches it.</p>
+
+<h3>2.2 GEMM — hipBLASLt heuristic check</h3>
+
+<p>Per the AMD ROCm 7.2.1 release notes, hipBLASLt 0.10's default heuristic for gfx942 BF16 / FP16 GEMMs is within ~5% of hand-tuned for the LoRA-rank-16-to-64 / hidden-2048-to-8192 shape range that mindXtrain hits. The probe runs a small check to confirm that 5% holds on the actual shapes; if it does, the plan locks in the default and moves on. Full heuristic enumeration — the brute-force search over hipBLASLt's algorithm space — is post-hackathon work. For the hackathon window, "the default is good enough and we measured it" is the right level of effort.</p>
+
+<h3>2.3 RCCL — collective topology resolution</h3>
+
+<p>RCCL handling is where the schema does most of the work. The 1-GPU path is a no-op: nothing collective happens, the plan's RCCL section is empty. The 8-GPU path sets <code>NCCL_MIN_NCHANNELS=112</code> and <code>GPU_MAX_HW_QUEUES=1</code> in the plan's env block, which are the values that consistently saturate xGMI on a full 8-card MI300X box. The 2- and 4-GPU paths <em>cannot reach the probe</em> — the schema rejected them at parse time. This is intentional. Asymmetric xGMI bandwidth between subsets of 2 or 4 MI300X cards silently bottlenecks FSDP shards, and it is the kind of bug that only shows up in throughput numbers nobody is looking at. A loud rejection at YAML-load time costs zero engineering hours; a silent 30% perf cliff costs two days.</p>
+
+<h2>3. Why AOT-only matters for production training</h2>
+
+<p>"AOT-only" is short for "ahead-of-time only — no JIT autotune in production." It is a one-line policy with surprisingly large consequences:</p>
+
+<table>
+<thead>
+<tr><th>Concern</th><th>JIT autotune behavior</th><th>AOT-only behavior</th></tr>
+</thead>
+<tbody>
+<tr><td>Determinism</td><td>Cold cache picks a different kernel each restart; loss curves drift across runs.</td><td>Same plan, same kernels, hash-equal outputs.</td></tr>
+<tr><td>Cold-start latency</td><td>First batch stalls while autotune searches.</td><td>First batch runs at steady-state.</td></tr>
+<tr><td>Reproducibility across machines</td><td>Different cache state, different decisions, different artifacts.</td><td>Plan ships with the artifact; another machine with the same plan trains the same way.</td></tr>
+<tr><td>Auditability</td><td>Decisions are runtime, ephemeral.</td><td>Decisions are a JSON file you can <code>cat</code>.</td></tr>
+<tr><td>Provenance</td><td>Manifest can't hash a runtime decision.</td><td>BLAKE3 hash of the plan goes into the manifest.</td></tr>
+</tbody>
+</table>
+
+<p>This is the cypherpunk2048 reproducibility standard applied to the ROCm reality. In a chain-anchored provenance pipeline, the autotune plan is part of the artifact, not part of the environment. Two independent operators with the same recipe and the same plan produce the same checkpoint. The receipt — <code>mindxtrain receipt &lt;manifest.json&gt; --config run.yaml</code> — verifies it.</p>
+
+<h2>4. The plan flowing through the pipeline</h2>
+
+<p>The training layer reads the plan, sets the env vars, and dispatches. Concretely:</p>
+
+<pre><code># Step 1: emit the plan (≤60s on MI300X)
+uv run mindxtrain bench --config qwen3_8b_sft_lora.yaml --out plan.json
+
+# Step 2: train, consuming the plan
+uv run mindxtrain train qwen3_8b_sft_lora.yaml --plan plan.json
+</code></pre>
+
+<p>Inside <code>mindxtrain/train/dispatch.py</code>, the plan determines:</p>
+
+<ul>
+<li>Which backend to dispatch to: Axolotl, Unsloth, torchtune, Primus-Turbo, or in-process TRL. Method-driven; SFT goes to Axolotl by default, GRPO/GSPO go to TRL, full-FSDP-32B goes to Primus-Turbo.</li>
+<li>Which env vars to set before subprocess-launching <code>accelerate</code>. The seven mandatory MI300X keys are baseline; the plan may override their values but never remove keys.</li>
+<li>Which Axolotl flags or torchtune CLI args correspond to the chosen attention backend.</li>
+</ul>
+
+<p>The plan is small, human-readable, BLAKE3-hashed into the manifest, and committed alongside the checkpoint. <code>mindxtrain receipt</code> re-hashes it on demand. There is no hidden state.</p>
+
+<h2>5. Budget and stretch — when 60 seconds isn't enough</h2>
+
+<p>Most recipes fit comfortably in a 60-second budget. The MoE recipes don't: <code>qwen3_30b_a3b_lora</code> and <code>qwen3_6_35b_a3b_lora</code> set <code>budget_seconds: 90</code> and <code>120</code> respectively, because expert imbalance means the probe has to time more shapes to make a credible decision. Even at 120 seconds, the probe is &lt;1% of a 4-hour training run's wall-clock, and the cost-amortization is favorable.</p>
+
+<p>The stretch path — for users who want to enumerate hipBLASLt heuristics or grid-search RCCL channel counts — is to run <code>mindxtrain bench --policy enumerate</code> once, save a richer plan, and reuse it across runs of the same shape. This stays AOT: the enumeration happens before training, the result is captured to disk, the loop never re-tunes. Same discipline, larger search.</p>
+
+<h2>6. Why no competitor framework ships this</h2>
+
+<p>The five major open training frameworks (Axolotl, Unsloth, torchtune, LLaMA-Factory, Primus-Turbo) each handle one slice of the problem. Axolotl is a great trainer-orchestrator. Unsloth is a great kernel-level optimizer. torchtune is a great PyTorch-native reference impl. Primus-Turbo is a great AMD-native trainer. None of them emit a static, hash-able, machine-portable AutotunePlan that the loop consumes verbatim. The reason is that they are framework-shaped: their job is to train. <em>The autotune layer is integration-shaped:</em> its job is to make a defensible kernel choice and capture it.</p>
+
+<p>mindXtrain's claim is that the integration layer is the product. The 60-second probe is the spine of the Application of Technology axis for the lablab judging — but more importantly, it is the spine of the project's reproducibility story. Without it, a Qwen3-8B fine-tune on an MI300X is "we trained it, here's the checkpoint, hopefully it works on your box too." With it, the checkpoint comes with a plan that says exactly which kernels were used and exactly which env vars were set, signed by a BLAKE3 hash and anchored to a write-once contract on Base.</p>
+
+<h2>7. Tomorrow</h2>
+
+<p>Day 3 is the actual LoRA fine-tune of <code>amd/Instella-3B-Instruct</code> on MI300X, using the plan that today's probe emitted. The dataset is curated and packed; the recipe is validated; the plan is BLAKE3'd. The training run produces a checkpoint directory, an <code>eval.json</code> from <code>lm-eval-harness</code>, and a quantized FP8 directory via AMD Quark. Day 5 is when the operator endpoint goes live and the demo URL exists. <a href="https://rage.pythai.net/mindxtrain-day-5-demo/">That post is here.</a></p>
+
+<hr>
+
+<h3>Related articles</h3>
+
+<ul>
+<li><a href="https://rage.pythai.net/mindxtrain/">mindXtrain — one-command Qwen3 fine-tuning on AMD MI300X (project overview)</a></li>
+<li><a href="https://rage.pythai.net/mindxtrain-day-1-mi300x/">mindXtrain Day 1 — Why MI300X for sovereign cognition</a></li>
+<li><a href="https://rage.pythai.net/mindxtrain-day-5-demo/">mindXtrain demo is live — Qwen3-8B on a single MI300X for less than $3</a></li>
+</ul>
+
+<p><em>Tagged <code>#AMDDevHackathon</code>. Code: <a href="https://github.com/codephreak/mindxtrain">github.com/codephreak/mindxtrain</a>. The 60-second AOT autotune probe lives in <code>mindxtrain/autotune/</code>; the schema enforcing AOT-only lives in <code>mindxtrain/config/schema.py</code>.</em></p>