Нафиг NVIDIA! Часть 2. Запустил 8 нейросетей на FPGA за $50 — и упёрся в DDR3

param(
    [int]$DDR = 1066,
    [int]$JOBS = 8,  # как нагрузить процессор 
    [int]$MCOLS = 4,
    [int]$VPU_CL = 2,
    [int]$LANES = 8,
    [int]$ACT_ROWS = 0,
    [int]$DSTEP = 1,
    [double]$CORE_MHZ = 0,
    [int]$PCIE_GEN = 2,
    [switch]$FAST,
    [switch]$ImplOnly,
    [string]$IMPL_STRATEGY = "",
    [string]$BUILD_ID = "00000000",

    # Измени этот путь под свою установку
    [string]$Vivado = "C:\Xilinx\Vivado\2024.2\bin\vivado.bat"
)

$ErrorActionPreference = "Stop"

$here = Split-Path -Parent $MyInvocation.MyCommand.Path
$root = (Resolve-Path (Join-Path $here "..\..")).Path
$out  = Join-Path $root "build\vivado"

if ($DDR -ne 1066) {
    throw "DDR3-$DDR is unsupported. LiteDRAM is generated for DDR3-1066."
}

if ($ACT_ROWS -eq 0) {
    $ACT_ROWS = $MCOLS
}

if ($CORE_MHZ -eq 0) {
    if ($FAST) {
        $CORE_MHZ = 100
    } else {
        $CORE_MHZ = 133.33
    }
}

if (!(Test-Path $Vivado)) {
    throw "Vivado not found: $Vivado"
}

New-Item -ItemType Directory -Force -Path $out | Out-Null

# Environment read by create_project.tcl / bd_native.tcl / build.tcl
$env:PCIE_GEN = "$PCIE_GEN"

if ($FAST) {
    $env:OTPU_FAST = "1"
} else {
    $env:OTPU_FAST = "0"
}

# Optional IP cache
$cache = Join-Path $env:LOCALAPPDATA "openTPU\vivado-ip-cache"
New-Item -ItemType Directory -Force -Path $cache | Out-Null
$env:OTPU_IP_CACHE = $cache

$createTcl = Join-Path $here "vivado\create_project.tcl"
$buildTcl  = Join-Path $here "vivado\build.tcl"

Write-Host ""
Write-Host "openTPU Vivado build"
Write-Host "-------------------"
Write-Host "Root       : $root"
Write-Host "Output     : $out"
Write-Host "DDR        : $DDR"
Write-Host "CORE_MHZ   : $CORE_MHZ"
Write-Host "MCOLS      : $MCOLS"
Write-Host "VPU_CL     : $VPU_CL"
Write-Host "LANES      : $LANES"
Write-Host "ACT_ROWS   : $ACT_ROWS"
Write-Host "DSTEP      : $DSTEP"
Write-Host "PCIe Gen   : $PCIE_GEN"
Write-Host "FAST       : $FAST"
Write-Host ""

if (!$ImplOnly) {

    Write-Host "=== CREATE PROJECT ==="

    & $Vivado `
        -mode batch `
        -nojournal `
        -log (Join-Path $out "create_project.log") `
        -source $createTcl `
        -tclargs `
            $DDR `
            $out `
            $MCOLS `
            $CORE_MHZ `
            $BUILD_ID `
            $VPU_CL `
            $LANES `
            $ACT_ROWS `
            $DSTEP

    if ($LASTEXITCODE -ne 0) {
        throw "Vivado create_project failed: $LASTEXITCODE"
    }

    Write-Host ""
    Write-Host "=== FULL BUILD ==="

    & $Vivado `
        -mode batch `
        -nojournal `
        -log (Join-Path $out "build.log") `
        -source $buildTcl `
        -tclargs `
            $out `
            $JOBS `
            "full" `
            $IMPL_STRATEGY

} else {

    Write-Host "=== IMPLEMENTATION ONLY ==="

    & $Vivado `
        -mode batch `
        -nojournal `
        -log (Join-Path $out "build_impl.log") `
        -source $buildTcl `
        -tclargs `
            $out `
            $JOBS `
            "impl" `
            $IMPL_STRATEGY
}

if ($LASTEXITCODE -ne 0) {
    throw "Vivado build failed: $LASTEXITCODE"
}

Write-Host ""
Write-Host "================================"
Write-Host "DONE"
Write-Host "Bitstream: $out\otpu.bit"
Write-Host "MCS      : $out\otpu.mcs"
Write-Host "Reports  : $out\reports"
Write-Host "================================"

#!/usr/bin/env bash
# openTPU hardware bring-up only: no LLM checkpoint download or inference.
set -Eeuo pipefail

PROJECT="$(pwd -P)"
RESET_VENV="${RESET_VENV:-1}"
VERBOSE="${VERBOSE:-1}"
DRIVER_URL="https://github.com/Xilinx/dma_ip_drivers.git"
PROJECT_URL="https://github.com/FeSens/openTPU.git"

mkdir -p "$PROJECT/logs" "$PROJECT/drivers" "$PROJECT/benchmarks" "$PROJECT/.local"
LOG="$PROJECT/logs/setup_$(date +%Y%m%d_%H%M%S).log"
exec > >(tee -a "$LOG") 2>&1
export PYTHONUNBUFFERED=1
export PS4='+ [${BASH_SOURCE##*/}:${LINENO}] '
[[ "$VERBOSE" == 1 ]] && set -x
trap 'rc=$?; set +x; printf "\nERROR: step failed on line %s (exit %s)\nLOG: %s\n" "$LINENO" "$rc" "$LOG" >&2' ERR
step() { printf '\n========== %s ==========\n' "$*"; }
die() { echo "ERROR: $*" >&2; exit 1; }

step 'Environment and project location'
echo "PROJECT=$PROJECT"; date -Is; uname -a
[[ "$PROJECT" != / && "$PROJECT" != "$HOME" ]] || die 'Run inside a dedicated openTPU directory, not / or HOME.'
command -v sudo >/dev/null || die 'sudo is required'
# A directory containing an unrelated project is not overwritten.
if [[ ! -f "$PROJECT/pyproject.toml" || ! -d "$PROJECT/opentpu" ]]; then
  if [[ -n "$(find "$PROJECT" -mindepth 1 -maxdepth 1 ! -name logs ! -name drivers ! -name benchmarks ! -name .local ! -name setup_opentpu_universal.sh ! -name setup.sh -print -quit)" ]]; then
    die 'Directory is not an openTPU checkout and is not empty. Run in an empty directory or an existing openTPU checkout.'
  fi
  step 'Install required packages for source checkout'
  sudo apt-get update
  sudo apt-get install -y git ca-certificates
  temp_checkout="$(mktemp -d)"
  git clone --depth 1 "$PROJECT_URL" "$temp_checkout/openTPU"
  cp -a "$temp_checkout/openTPU/." "$PROJECT/"
  rm -rf -- "$temp_checkout"
fi
[[ -d "$PROJECT/opentpu" && -f "$PROJECT/pyproject.toml" ]] || die 'openTPU sources not found'
if git -C "$PROJECT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
  git -C "$PROJECT" rev-parse HEAD || true
  git -C "$PROJECT" status --short || true
fi

step 'Host packages'
sudo apt-get update
sudo apt-get install -y build-essential make gcc git rsync pciutils kmod python3 python3-venv python3-pip "linux-headers-$(uname -r)"

step 'Detect Xilinx PCIe device automatically'
mapfile -t xilinx_devices < <(lspci -Dnnd 10ee: | awk '{print $1}')
((${#xilinx_devices[@]} > 0)) || die 'No Xilinx PCIe devices found (vendor 10ee). Check power, bitstream and PCIe enumeration.'
printf 'Xilinx devices found:\n'; lspci -Dnn -d 10ee:
if [[ -n "${PCI_BDF:-}" ]]; then
  BDF="$PCI_BDF"
  [[ "$BDF" == 0000:* ]] || BDF="0000:$BDF"
else
  # Prefer the openTPU/ XDMA endpoint ID 10ee:7028; otherwise accept only a unique Xilinx endpoint.
  mapfile -t preferred < <(lspci -Dnnd 10ee:7028 | awk '{print $1}')
  if ((${#preferred[@]} == 1)); then
    BDF="${preferred[0]}"
  elif ((${#xilinx_devices[@]} == 1)); then
    BDF="${xilinx_devices[0]}"
  else
    die 'Several Xilinx devices found. Rerun with PCI_BDF=0000:BB:DD.F ./setup_opentpu_universal.sh'
  fi
fi
[[ -d "/sys/bus/pci/devices/$BDF" ]] || die "PCI device $BDF does not exist"
echo "SELECTED_PCI_BDF=$BDF"
lspci -nnk -s "$BDF"; lspci -vv -s "$BDF" | grep -E 'LnkCap:|LnkSta:|Region [0-9]:' || true
for item in vendor device subsystem_vendor subsystem_device current_link_speed current_link_width max_link_speed max_link_width; do
  file="/sys/bus/pci/devices/$BDF/$item"; [[ -r "$file" ]] && printf '%s=' "$item" && cat "$file"
done

step 'Get XDMA sources inside the project'
DRIVER_REPO="$PROJECT/drivers/dma_ip_drivers"
if [[ ! -f "$DRIVER_REPO/XDMA/linux-kernel/xdma/Makefile" ]]; then
  if [[ -f "$HOME/dma_ip_drivers/XDMA/linux-kernel/xdma/Makefile" ]]; then
    mkdir -p "$DRIVER_REPO"
    rsync -a "$HOME/dma_ip_drivers/" "$DRIVER_REPO/"
  else
    [[ ! -e "$DRIVER_REPO" || -z "$(ls -A "$DRIVER_REPO" 2>/dev/null)" ]] || die "Driver directory $DRIVER_REPO contains unrelated files"
    git clone "$DRIVER_URL" "$DRIVER_REPO"
  fi
fi
[[ -f "$DRIVER_REPO/XDMA/linux-kernel/include/libxdma_api.h" ]] || die 'Missing XDMA header include/libxdma_api.h'

step 'Compile driver on local filesystem (source remains in project)'
# Kernel module builds frequently fail on SMB shares: use a local temp build tree.
BUILD_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/opentpu-xdma.XXXXXXXX")"
trap 'rc=$?; set +x; echo "Exit $rc; log: $LOG" >&2; [[ -n "${BUILD_ROOT:-}" && -d "$BUILD_ROOT" ]] && rm -rf -- "$BUILD_ROOT"; exit "$rc"' EXIT
rsync -a --exclude='.git' "$DRIVER_REPO/" "$BUILD_ROOT/dma_ip_drivers/"
cd "$BUILD_ROOT/dma_ip_drivers/XDMA/linux-kernel/xdma"
make clean
make -j"$(nproc)" KCFLAGS="-I$(realpath ../include)"
[[ -s xdma.ko ]] || die 'xdma.ko not created'
modinfo ./xdma.ko | grep -E '^(filename|vermagic|description):' || true
[[ "$(modinfo -F vermagic ./xdma.ko)" == "$(uname -r) "* ]] || die 'Built module vermagic does not match running kernel'
mkdir -p "$PROJECT/drivers/modules/$(uname -r)"
cp -f ./xdma.ko "$PROJECT/drivers/modules/$(uname -r)/xdma.ko"

step 'Load compiled XDMA (no reboot, no flash programming)'
if [[ -L "/sys/bus/pci/devices/$BDF/driver" ]]; then
  active_driver="$(basename "$(readlink -f "/sys/bus/pci/devices/$BDF/driver")")"
  [[ "$active_driver" == xdma ]] || die "Selected board is bound to $active_driver; refusing to replace it automatically"
fi
if lsmod | awk '$1=="xdma" {found=1} END {exit !found}'; then
  sudo modprobe -r xdma || die 'XDMA in use, stop programs that access /dev/xdma* and retry'
fi
sudo insmod "$PROJECT/drivers/modules/$(uname -r)/xdma.ko" poll_mode=0
sleep 2
lspci -nnk -s "$BDF"
ls -l /dev/xdma* || true
mapfile -t udev_nodes < <(compgen -G '/dev/xdma*_user' || true)
((${#udev_nodes[@]} >= 1)) || die 'No /dev/xdma*_user. Inspect kernel log.'
DEVICE="${DEVICE:-${udev_nodes[0]%_user}}"
[[ -c "${DEVICE}_user" && -c "${DEVICE}_h2c_0" && -c "${DEVICE}_c2h_0" ]] || die "Missing DMA nodes for $DEVICE"
echo "SELECTED_XDMA=$DEVICE"
sudo dmesg | tail -45 || true

step 'Create Python environment inside project'
cd "$PROJECT"
VENV="$PROJECT/.venv"
# SMB/CIFS can break virtual environments (symlinks, executable permissions).
# Never silently relocate .venv outside the project.
if [[ -L "$VENV" ]]; then
  rm -- "$VENV"
elif [[ -e "$VENV" && "$RESET_VENV" == 1 ]]; then
  [[ -d "$VENV" ]] || die "Unexpected .venv path: $VENV"
  rm -rf -- "$VENV"
fi
if [[ ! -x "$VENV/bin/python" ]]; then
  python3 -m venv --copies "$VENV" || die "Cannot create .venv on this filesystem; mount with Linux-compatible permissions or use a local checkout"
fi
[[ -x "$VENV/bin/python" ]] || die "Python in $VENV is not executable (SMB mount may be noexec)"
"$VENV/bin/python" -m pip install -U pip setuptools wheel transformers tokenizers safetensors sentencepiece protobuf torch  https://download.pytorch.org/whl/cpu
"$VENV/bin/python" -m pip install -e "$PROJECT"
export PATH="$VENV/bin:$PATH"
command -v otpu-selftest; command -v otpu-diag; command -v otpu-smi

step 'Software CLI smoke checks'
otpu-diag --help >/dev/null
otpu-selftest --help >/dev/null
otpu-smi --help >/dev/null

step 'FPGA board diagnostics (no model / no LLM)'
otpu-diag --dev "$DEVICE"

step 'SUCCESS: board bring-up complete, no LLM downloaded or executed'
echo "Project: $PROJECT"
echo "PCIe FPGA: $BDF"
echo "XDMA: $DEVICE"
echo "Venv: $VENV"
echo "Log: $LOG"
echo 'No Qwen model was downloaded, loaded or launched.'

Если посмотреть глубже, архитектура openTPU состоит из нескольких подсистем.

#!/usr/bin/env python3
"""openTPU Kintex-7 comprehensive benchmark: serial, resumable, non-destructive.
Uses otpu-chat + native otpu-smi --json -l 0.5. Does NOT flash or change RTL.
"""
import argparse, csv, datetime, hashlib, json, os, pathlib, platform, re, shlex, shutil, signal, subprocess, sys, time

ROOT = pathlib.Path.home() / 'openTPU-main'
# Actual folder names already used by this project. Case insensitive resolution also supported.
MODELS = [
 ('lfm230','LFM2.5-230M','dense'), ('qwen06','Qwen3-0.6B','dense'),
 ('qwen08','Qwen3.5-0.8B','dense'), ('qwen17','Qwen3-1.7B','experimental'),
 ('qwen2','Qwen3.5-2B','dense'), ('lfm26','LFM2-2.6B','dense'),
 ('smol3','SmolLM3-3B','dense'), ('phi4','Phi-4-mini-instruct','dense'),
 ('qwen4','Qwen3.5-4B','dense'), ('gemmaE2','gemma-4-E2B-it','special'),
 ('gemmaE4','gemma-4-E4B-it','special'), ('lfm8','LFM2.5-8B-A1B','moe'),
]
# Only model/format pairs known to fit in card's 4GiB image or supported by the project.
# "default" lets openTPU choose its model-specific mixed layout (Gemma).
FP4_FORMATS = [('fp4','int8')]
INT8_OK = {'lfm230','qwen06','qwen08','qwen2','lfm26','smol3','phi4'}
FP4_ONLY = {'qwen4','qwen17','lfm8'}
TTFT = re.compile(r'TTFT\s+([\d.]+)s;')
PREFILL = re.compile(r'prefill\s+(\d+)\s+tokens,\s+([\d.]+)\s+\(device\s+([\d.]+)\)\s+tok/s')
DECODE = re.compile(r'decode\s+(\d+)\s+tokens,\s+([\d.]+)\s+\(device\s+([\d.]+)\)\s+tok/s,\s+([\d.]+)\s+Mcycles/token')
NUM = (int,float)
FIELDS = ['id','model','group','scenario','wformat','head_format','prompt_target_words',
 'actual_prefill_tokens','cap','max_new','repeat','status','exit_code','elapsed_s',
 'ttft_s','prefill_wall_tps','prefill_device_tps','decode_tokens','decode_wall_tps',
 'decode_device_tps','mcycles_per_token','cache_files_before','cache_files_after',
 'cache_bytes_before','cache_bytes_after','smi_samples','smi_active_samples',
 'smi_running_time_s','dram_read_gbs','dram_write_gbs','dram_total_gbs',
 'dram_read_peak_gbs','dram_write_peak_gbs','mxu_busy_pct','mxu_mac_pct',
 'vpu_busy_pct','qnt_busy_pct','dma_busy_pct','tmem_deny_pct','dram_wait_pct',
 'mxu_starve_pct','temperature_mean_c','temperature_max_c', 'power_mean_w',
 'error','folder']
COUNTER_KEYS = ('RUNNING','MXU_BUSY','MXU_MAC','VPU_BUSY','QNT_BUSY',
                'DMA_BUSY','TMEM_DENY','DRAM_WAIT','MXU_STARVE')

def now(): return datetime.datetime.now().isoformat(timespec='seconds')
def jsondump(path,obj):
    tmp=path.with_suffix(path.suffix+'.tmp')
    tmp.write_text(json.dumps(obj,ensure_ascii=False,indent=2,default=str),encoding='utf-8')
    tmp.replace(path)
def models_present(root):
    base=root/'models'
    if not base.exists():return {}
    candidates={p.name.lower():p for p in base.iterdir() if p.is_dir()}
    return {key:candidates.get(name.lower()) for key,name,_ in MODELS}
def valid_checkpoint(p):
    if not p or not (p/'config.json').is_file(): return False
    return bool(list(p.glob('*.safetensors')) or list(p.glob('*.bin')))
def cache_snapshot(path):
    try:
        return {'files':sum(1 for p in path.rglob('*') if p.is_file()),
                'bytes':sum(p.stat().st_size for p in path.rglob('*') if p.is_file())}
    except (OSError,PermissionError): return {'files':None,'bytes':None}
def iter_json(path):
    dec=json.JSONDecoder()
    if not path.exists():return
    with path.open('r',errors='replace') as f:
        buf=''
        while True:
            data=f.read(262144)
            if data:buf+=data
            pos=0
            while pos<len(buf):
                while pos<len(buf) and buf[pos].isspace():pos+=1
                if pos==len(buf):break
                try: obj,end=dec.raw_decode(buf,pos)
                except ValueError:break
                pos=end
                if isinstance(obj,list):
                    for item in obj:
                        if isinstance(item,dict):yield item
                elif isinstance(obj,dict):yield obj
            buf=buf[pos:]
            if not data:break

def telemetry(path):
    rec=list(iter_json(path)); utilrows=[]
    for r in rec:
        u=r.get('util') or {}
        if not isinstance(u,dict):continue
        # running is a status AT sample end; RUNNING fractional utilization detects
        # sub-500ms kernel launches and is more reliable for LLM token-by-token decode.
        run=u.get('RUNNING')
        sec=(r.get('sample') or {}).get('seconds')
        if isinstance(run,NUM) and run>0 and isinstance(sec,NUM) and sec>0:
            utilrows.append((r, float(sec)*float(run)))
    out={'smi_samples':len(rec),'smi_active_samples':len(utilrows),
         'smi_running_time_s':round(sum(w for _,w in utilrows),5)}
    # Weighted average for active device-cycle metrics. Each r['util'][k] is a
    # wall-time fraction over sampling period; dividing by RUNNING gives percent
    # of cycles running, clipped only for occasional sampling noise.
    run_s=out['smi_running_time_s']
    if run_s>0:
        for k in COUNTER_KEYS[1:]:
            vals=[]
            for r,w in utilrows:
                u=r['util']; sec=r['sample']['seconds']
                if isinstance(u.get(k),NUM): vals.append((float(u[k])*sec, w))
            if vals:
                val=100*sum(x for x,_ in vals)/sum(w for _,w in vals)
                out[k.lower()+'_pct']=round(val,3)
        for source,target in [('dram_rd_gbs','dram_read_gbs'),
                              ('dram_wr_gbs','dram_write_gbs'),
                              ('dram_gbs','dram_total_gbs')]:
            vals=[]
            for r,w in utilrows:
                val=r.get(source)
                if isinstance(val,NUM): vals.append((float(val)*r['sample']['seconds'],w))
            if vals:out[target]=round(sum(v for v,_ in vals)/sum(w for _,w in vals),4)
    for source,target in [('dram_rd_gbs','dram_read_peak_gbs'),
                          ('dram_wr_gbs','dram_write_peak_gbs')]:
        vals=[r[source] for r,_ in utilrows if isinstance(r.get(source),NUM)]
        if vals:out[target]=round(max(vals),4)
    temps=[r.get('temp_c') for r in rec if isinstance(r.get('temp_c'),NUM)]
    if temps:
        out['temperature_mean_c']=round(sum(temps)/len(temps),3)
        out['temperature_max_c']=round(max(temps),3)
    # Power is optional and not available with --no-i2c on some cards.
    power=[r.get('power') for r in rec if isinstance(r.get('power'),NUM)]
    if power:out['power_mean_w']=round(sum(power)/len(power),3)
    return out

def prompt_words(n):
    # Repeated meaningful English text. "n" is an approximate WORD target,
    # NOT a guaranteed token count; the device's actual prefill count is recorded.
    base=('Explain field programmable gate arrays, matrix multiplication, neural network '
          'inference, quantized weights, memory bandwidth, PCI Express, and streaming '
          'computations. Discuss real hardware bottlenecks with practical examples. ')
    words=(base*max(1, (n//20)+2)).split()
    return ' '.join(words[:n]) + ' Continue with a detailed technical explanation.'

def matrix(selected,profile,format_test,include_moe):
    # Stable deterministic order, group by MODEL to reuse host quant cache.
    for key,name,group in MODELS:
        if key not in selected:continue
        if group=='moe' and not include_moe:continue
        forms=[(None,None)] if group=='special' else list(FP4_FORMATS)
        if group=='special' and key=='gemmaE2':forms=[('fp4','int8'),('fp4','fp4')]
        if format_test and key in INT8_OK:forms.append(('int8','int8'))
        for wf,hf in forms:
            label=(wf or 'default')+'_'+(hf or 'default')
            # quick=2 tests/model, standard=6-8 tests/model, full=14+ tests/model
            scenarios=[('baseline',64,256,1024,1),('warm',64,256,1024,2)]
            if profile in ('standard','full'):
                scenarios += [('short_prefill',16,256,1024,1),
                              ('long_prefill',128,256,1024,1),
                              ('short_decode',64,64,1024,1),
                              ('long_decode',64,512,1024,1)]
            if profile=='full':
                scenarios += [('prefill_128',128,256,1024,1),
                              ('prefill_512',512,256,2048,1),
                              ('decode_1024',64,1024,2048,1),
                              ('cap_512',64,256,512,1),
                              ('cap_2048',64,256,2048,1)]
            for typ,words,new,cap,repeat in scenarios:
                yield {'id':f'{key}_{label}_{typ}','key':key,'model':name,'group':group,
                       'scenario':typ,'wformat':wf or 'default',
                       'head_format':hf or 'default','prompt_target_words':words,
                       'cap':cap,'max_new':new,'repeat':repeat}

def terminate(proc):
    if proc is None or proc.poll() is not None:return
    try:proc.send_signal(signal.SIGINT);proc.wait(timeout=3)
    except (OSError,subprocess.TimeoutExpired):
        proc.terminate()
        try:proc.wait(timeout=4)
        except subprocess.TimeoutExpired:proc.kill();proc.wait()

def benchmark(case,root,folder,env,args,path):
    row={k:v for k,v in case.items() if k in FIELDS}; row['folder']=str(folder)
    model_dir=path[case['key']]
    if not valid_checkpoint(model_dir):
        row.update(status='missing',error='Missing complete checkpoint/config; no download is attempted')
        return row
    folder.mkdir(parents=True,exist_ok=True)
    cmd=[str(root/'.venv/bin/otpu-chat'),'--backend','board','--dev',args.device,
         '--model',str(model_dir),'--cap',str(case['cap']), '--max-new',
         str(case['max_new']),'--greedy','--prompt',prompt_words(case['prompt_target_words'])]
    if case['wformat']!='default':cmd.extend(['--wformat',case['wformat']])
    if case['head_format']!='default':cmd.extend(['--head-format',case['head_format']])
    meta={'command':cmd,'started_at':now(), 'env':{k:env.get(k) for k in
          ('OTPU_IMAGE_CACHE','OTPU_DMA_CHUNK_KIB','OPENBLAS_NUM_THREADS')},
          'checkpoint':str(model_dir)}
    jsondump(folder/'command.json',meta)
    cb=cache_snapshot(pathlib.Path(env['OTPU_IMAGE_CACHE']))
    row['cache_files_before']=cb['files'];row['cache_bytes_before']=cb['bytes']
    start=time.monotonic(); smi=None
    try:
        with (folder/'smi.json').open('w',buffering=1) as sf, \
             (folder/'smi.stderr.log').open('w') as se, \
             (folder/'chat.log').open('w',buffering=1) as cf:
            smi=subprocess.Popen([str(root/'.venv/bin/otpu-smi'),'--dev',args.device,
                                  '--no-i2c','--json','-l',str(args.interval)],
                                 cwd=root,env=env,stdout=sf,stderr=se)
            time.sleep(min(0.2,args.interval))
            try:
                result=subprocess.run(cmd,cwd=root,env=env,stdout=cf,
                                      stderr=subprocess.STDOUT,timeout=args.timeout)
                row['exit_code']=result.returncode
                row['status']='ok' if result.returncode==0 else 'failed'
            except subprocess.TimeoutExpired:
                row.update(status='timeout',exit_code=-1,error='otpu-chat timed out')
            finally:terminate(smi)
    except Exception as e:
        terminate(smi)
        row.update(status='failed',error=repr(e))
    row['elapsed_s']=round(time.monotonic()-start,3)
    ca=cache_snapshot(pathlib.Path(env['OTPU_IMAGE_CACHE']))
    row['cache_files_after']=ca['files'];row['cache_bytes_after']=ca['bytes']
    content=(folder/'chat.log').read_text(errors='replace') if (folder/'chat.log').exists() else ''
    for regex,keys in ((TTFT,['ttft_s']),
                       (PREFILL,['actual_prefill_tokens','prefill_wall_tps','prefill_device_tps']),
                       (DECODE,['decode_tokens','decode_wall_tps','decode_device_tps','mcycles_per_token'])):
        mat=regex.search(content)
        if mat:
            for k,v in zip(keys,mat.groups()):row[k]=int(v) if k in ('actual_prefill_tokens','decode_tokens') else float(v)
    try:row.update(telemetry(folder/'smi.json'))
    except Exception as e:row['error']=(row.get('error','')+'; SMI parse: '+repr(e))
    if row.get('status')=='ok' and not row.get('decode_tokens'):
        row['status']='no_metrics';row['error']='Process returned 0 but decode metrics not found; check chat.log'
    jsondump(folder/'result.json',row)
    return row

def main():
    ap=argparse.ArgumentParser(description=__doc__)
    ap.add_argument('--root',type=pathlib.Path,default=ROOT)
    ap.add_argument('--device',default='/dev/xdma0')
    ap.add_argument('--profile',choices=['quick','standard','full'],default='standard')
    ap.add_argument('--models',nargs='+',default=['lfm230','qwen06','qwen08','qwen2','lfm26','smol3','phi4','qwen4'],
                    help='Short model IDs; see --list')
    ap.add_argument('--include-moe',action='store_true')
    ap.add_argument('--all-formats',action='store_true',help='Enable INT8 comparison where known')
    ap.add_argument('--timeout',type=int,default=1800)
    ap.add_argument('--interval',type=float,default=0.5)
    ap.add_argument('--output',type=pathlib.Path)
    ap.add_argument('--list',action='store_true')
    ap.add_argument('--dry-run',action='store_true')
    args=ap.parse_args();root=args.root.expanduser().resolve();paths=models_present(root)
    valid_ids={k for k,_,_ in MODELS}
    invalid=set(args.models)-valid_ids
    if invalid:ap.error('Unknown model IDs: '+', '.join(sorted(invalid)))
    if args.list:
        for k,n,g in MODELS:print(f'{k:10} {g:14} {"READY" if valid_checkpoint(paths[k]) else "MISSING":8} {n}')
        return
    if not args.include_moe and 'lfm8' in args.models:
        ap.error('For MoE add --include-moe (pool generation and offload may be slow)')
    cases=list(matrix(set(args.models),args.profile,args.all_formats,args.include_moe))
    print('Planned:',len(cases),'tests (sequential).')
    if args.dry_run:
        for i,c in enumerate(cases,1):print(f'{i:3} {c["id"]}: cap={c["cap"]} new={c["max_new"]} words={c["prompt_target_words"]}')
        return
    for binary in ('otpu-chat','otpu-smi'):
        if not (root/'.venv/bin'/binary).exists():ap.error('Missing '+binary+' in '+str(root/'.venv/bin'))
    # --device is a logical XDMA prefix; Linux creates /dev/xdma0_user,
    # /dev/xdma0_h2c_0, /dev/xdma0_c2h_0 (not /dev/xdma0).
    device_prefix = pathlib.Path(args.device)
    if not device_prefix.exists():
        required = [pathlib.Path(args.device + suffix)
                    for suffix in ('_user', '_h2c_0', '_c2h_0')]
        missing = [str(path) for path in required if not path.is_char_device()]
        if missing:
            ap.error('XDMA device nodes missing: ' + ', '.join(missing))
    env=os.environ.copy()
    env.setdefault('OTPU_IMAGE_CACHE',str(pathlib.Path.home()/'otpu-qcache'))
    env.setdefault('OTPU_DMA_CHUNK_KIB','1024')
    env.update(OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',OMP_NUM_THREADS='1')
    out=(args.output or root/'benchmarks'/'llm'/('all_'+datetime.datetime.now().strftime('%Y%m%d_%H%M%S'))).expanduser().resolve()
    out.mkdir(parents=True,exist_ok=True)
    inventory={'created_at':now(),'hostname':platform.node(),'platform':platform.platform(),
               'python':sys.version,'profile':args.profile,'device':args.device,
               'model_directories':{k:str(v) if v else None for k,v in paths.items()},
               'quant_image_cache':env['OTPU_IMAGE_CACHE'],'dma_chunk_kib':env['OTPU_DMA_CHUNK_KIB'],
               'cases_planned':len(cases),'notes':[
                   'Native otpu-smi JSON; RUNNING util fraction detects sub-interval kernels.',
                   'DRAM/MXU active-cycle estimates exclude idle intervals; no phase tagging.',
                   'Prompts specified by words; actual tokens taken from otpu-chat log.',
                   'FP4 cache persists across invocations; every process may reupload board image.',
                   'Temperature includes host quantization/loading time; I2C power disabled.',
                   'Missing metrics are blank, never zero-filled.']}
    jsondump(out/'manifest.json',inventory)
    results={}
    if (out/'results.jsonl').exists():
        for line in (out/'results.jsonl').read_text().splitlines():
            try:r=json.loads(line);results[r['id']]=r
            except (ValueError,KeyError):pass
    for i,case in enumerate(cases,1):
        if case['id'] in results:
            print(f'[{i}/{len(cases)}] SKIP completed {case["id"]}',flush=True)
            continue
        print(f'[{i}/{len(cases)}] {now()} {case["id"]}',flush=True)
        row=benchmark(case,root,out/case['id'],env,args,paths)
        results[case['id']]=row
        with (out/'results.jsonl').open('a') as f:f.write(json.dumps(row,ensure_ascii=False)+'\n')
        with (out/'summary.csv').open('w',newline='') as f:
            w=csv.DictWriter(f,FIELDS,extrasaction='ignore');w.writeheader()
            for c in cases:
                if c['id'] in results:w.writerow(results[c['id']])
        jsondump(out/'summary.json',[results[c['id']] for c in cases if c['id'] in results])
        print('     ',row['status'],row.get('decode_wall_tps','—'),'tok/s',
              'DDR3 RD',row.get('dram_read_gbs','—
                                '),'GB/s',
              'Tmax',row.get('temperature_max_c','—'),'C',flush=True)
        if row['status']=='timeout':
            print('Timeout: stopping suite to avoid hardware/driver issues. Resume with same --output.',file=sys.stderr)
            break
    print('Results:',out,flush=True)
if __name__=='__main__':main()

#!/usr/bin/env python3
"""Real-board openTPU GEMM benchmark. No LLM, no RTL simulation, no bitstream change.

Uses runtime.compile_kernel() -> ISA assembler -> Board.load_program()/run().
Counts device cycles from hardware R_CYCLES; separately measures host wall time.
"""
import argparse
import csv
import json
import math
import sys
import time
from pathlib import Path

import numpy as np

from opentpu import isa as I
from opentpu import language as ol
from opentpu.host.board import Board, XdmaTransport, device_config
from opentpu.runtime import Input, Output, Weight, compile_kernel


@ol.jit
def gemm(x, w, out):
    # x[M,K] * w[N,K].T -> y[M,N]; W8A8, block-quantized MXU.
    q = ol.quantize(ol.load(x))
    ol.store(out, ol.dot(q, w))


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument('--dev', default='/dev/xdma0')
    ap.add_argument('--sizes', default='128,512,1024,2048', help='N=K sizes, multiples of D=128')
    ap.add_argument('--rows', default='1,4,8', help='M rows')
    ap.add_argument('--repeats', type=int, default=5)
    ap.add_argument('--max-rel-error', type=float, default=0.08,
                    help='Relative L2 norm error vs fp32 NumPy (quantized W8A8)')
    ap.add_argument('--out', default=None, help='Results directory')
    a = ap.parse_args()
    if a.repeats < 1:
        ap.error('--repeats must be >= 1')
    root = Path(a.out or ('benchmarks/gemm_' + time.strftime('%Y%m%d_%H%M%S')))
    root.mkdir(parents=True, exist_ok=True)
    csv_path = root / 'results.csv'
    config_path = root / 'board.json'
    sizes = [int(v) for v in a.sizes.split(',')]
    rows = [int(v) for v in a.rows.split(',')]
    print('openTPU physical FPGA GEMM', flush=True)
    print(f'Device: {a.dev}; Results: {root}', flush=True)

    with Board(XdmaTransport(a.dev)) as board:
        info = board.info()
        # Restrict *host-side staging image* to 16 MiB, not physical installed DRAM.
        # The actual board keeps its full DDR3; our data and code occupy only this area.
        cfg = device_config(info, DRAM_BYTES=16 << 20)
        core_hz = info['core_khz'] * 1000
        config_path.write_text(json.dumps({
            'device': a.dev, 'core_hz': core_hz, 'D': cfg.D, 'MCOLS': cfg.MCOLS,
            'LANES': cfg.LANES, 'DRAM_BYTES_host_image': cfg.DRAM_BYTES,
            'sizes': sizes, 'rows': rows, 'repeats': a.repeats,
            'note': 'GOPS assumes 2 operations per MAC; W8A8 quantized GEMM'
        }, indent=2))
        if not info['calibrated']:
            raise RuntimeError('DDR3 not calibrated')
        print(f"Core {core_hz/1e6:.3f} MHz, D={cfg.D}, MCOLS={cfg.MCOLS}, "
              f"LANES={cfg.LANES}; ECC calibrated={info['calibrated']}", flush=True)
        print('FPGA locked. Compiling and running hardware GEMM...', flush=True)
        fieldnames = ('M,N,K,repetition,core_cycles,core_ms,host_run_ms,readback_ms,ops,gops_device,rel_l2_error,max_abs_error,verified').split(',')
        failures = 0
        with csv_path.open('w', newline='') as f:
            writer = csv.DictWriter(f, fieldnames=fieldnames)
            writer.writeheader()
            for N in sizes:
                K = N
                if K % cfg.D or N % cfg.D:
                    raise ValueError(f'N,K must be multiples of D={cfg.D}')
                for M in rows:
                    if M <= 0 or M > 8:
                        raise ValueError('M must be 1..8 for this small-TMEM test')
                    rng = np.random.default_rng(1000000 + M * 10000 + N)
                    x = rng.normal(size=(M, K)).astype(np.float32)
                    w = (rng.normal(size=(N, K)) / np.sqrt(K)).astype(np.float32)
                    t_compile = time.perf_counter()
                    compiled, images = compile_kernel(
                        gemm, cfg, x=Input(x), w=Weight(w, shard=0, fmt='int8'),
                        out=Output((M, N)))
                    instructions = I.assemble(compiled.programs[0])
                    # Program stored outside the runtime's data image, inside 16 MiB.
                    program_addr = 12 << 20
                    program_bytes = len(instructions) * 4
                    if program_addr + program_bytes > cfg.DRAM_BYTES:
                        raise RuntimeError('Program exceeds reserved DRAM image')
                    # The runtime's allocations start at 0 and stay below 8 MiB for these sizes.
                    # Host image includes initialized zeros for ECC-safe subsequent reads.
                    if (8 << 20) > program_addr:
                        raise RuntimeError('Invalid layout')
                    if np.any(images[0][8 << 20:program_addr]):
                        raise RuntimeError('Data image crosses 8 MiB region; reduce N/K')
                    board.write(0, images[0][:8 << 20])
                    board.load_program(program_addr, instructions)
                    result_addr = compiled.layout.params['out'][1]
                    if result_addr + M * N * 4 > (8 << 20):
                        raise RuntimeError('Output exceeds initialized 8 MiB')
                    ref = x @ w.T
                    print(f'M={M:2d}, N=K={N:4d}: compiled in '
                          f'{time.perf_counter()-t_compile:.2f}s, program {len(instructions)//8} instr',
                          flush=True)
                    for rep in range(1, a.repeats + 1):
                        t0 = time.perf_counter()
                        stats = board.run()
                        host_s = time.perf_counter() - t0
                        t1 = time.perf_counter()
                        got = board.read(result_addr, M * N * 4).view(np.float32).reshape(M, N).copy()
                        read_s = time.perf_counter() - t1
                        err = float(np.linalg.norm((got-ref).astype(np.float64)) /
                                    max(float(np.linalg.norm(ref.astype(np.float64))), 1e-12))
                        abserr = float(np.max(np.abs(got-ref)))
                        valid = bool(np.isfinite(got).all() and err <= a.max_rel_error)
                        failures += not valid
                        cycles = int(stats['cycles'])
                        ops = 2 * M * N * K
                        device_s = cycles / core_hz
                        gops = ops / device_s / 1e9 if cycles else float('nan')
                        writer.writerow(dict(M=M, N=N, K=K, repetition=rep, core_cycles=cycles,
                            core_ms=round(device_s*1e3,6), host_run_ms=round(host_s*1e3,6),
                            readback_ms=round(read_s*1e3,6), ops=ops,
                            gops_device=round(gops,6), rel_l2_error=round(err,7),
                            max_abs_error=round(abserr,7), verified=int(valid)))
                        f.flush()
                        print(f'  #{rep}: {cycles} cycles, {device_s*1e3:.3f} ms device, '
                              f'{gops:.3f} GOPS, error={err:.5f}, '
                              f"{'PASS' if valid else 'FAIL'}", flush=True)
    print(f'\nResults CSV: {csv_path}; mismatches: {failures}')
    return 1 if failures else 0


if __name__ == '__main__':
    try:
        sys.exit(main())
    except Exception as e:
        print(f'ERROR: {type(e).__name__}: {e}', file=sys.stderr)
        raise