pull-hf-model: support datasets and spaces, not just models
Adds optional repo_type var (default "model"; valid: model/dataset/space)
that's forwarded to `hf download --repo-type` and threaded through the
verify step (cache-dir prefix tracks repo_type: models--, datasets--,
spaces--).
The playbook was previously model-only because hf download defaults
repo_type=model — pulling a dataset surfaced as a 404 against the wrong
HF API path. Now generic.
Smoke-tested 2026-05-13:
scripts/elway ana-ml2 --playbook playbooks/pull-hf-model.yaml \
--var hf_repo=Skywork/Skywork-Reward-Preference-80K-v0.2 \
--var repo_type=dataset
→ 200 MB parquet cached at
/tank/aimodels/huggingface/hub/datasets--Skywork--Skywork-Reward-Preference-80K-v0.2/
Filename is misleading now (playbook handles more than models); leaving
the rename for a future cleanup since pull-hf-model.yaml is referenced
from the llama-swap config.yaml's 2026-05-13 changelog entry.
This commit is contained in:
@@ -1,16 +1,17 @@
|
|||||||
# Pull a model from HuggingFace into ana-ml2's shared HF cache
|
# Pull a model or dataset from HuggingFace into ana-ml2's shared HF cache
|
||||||
# (`/tank/aimodels/huggingface/`). Model-format-agnostic — handles GGUFs,
|
# (`/tank/aimodels/huggingface/`). Repo-type-agnostic — handles GGUFs,
|
||||||
# safetensors, or anything else HF hosts.
|
# safetensors, datasets, or anything else HF hosts.
|
||||||
#
|
#
|
||||||
# Consumers (llama-swap, vLLM, others) read from the same cache:
|
# Consumers (llama-swap, vLLM, training jobs) read from the same cache:
|
||||||
# - llama-swap's config.yaml references entries via `-hf <user>/<repo>`
|
# - llama-swap's config.yaml references entries via `-hf <user>/<repo>`
|
||||||
# - vLLM stacks reference via `--model <user>/<repo>` with HF_HOME mounted
|
# - vLLM stacks reference via `--model <user>/<repo>` with HF_HOME mounted
|
||||||
|
# - datasets.load_dataset() reads from HF_HOME/datasets too
|
||||||
#
|
#
|
||||||
# This playbook does NOT edit any consumer config. Run it for the download,
|
# This playbook does NOT edit any consumer config. Run it for the download,
|
||||||
# then hand-edit the relevant `stacks/<consumer>/conf/*.yaml` and deploy
|
# then hand-edit the relevant `stacks/<consumer>/conf/*.yaml` and deploy
|
||||||
# that consumer separately. Keeps the download mechanical and the
|
# that consumer separately. Keeps the download mechanical and the
|
||||||
# per-model params (ctx-size, quant choice, sampler defaults, etc.) where
|
# per-asset params (ctx-size, quant choice, sampler defaults, split filters,
|
||||||
# they belong — in human-curated config.
|
# etc.) where they belong — in human-curated config.
|
||||||
#
|
#
|
||||||
# Usage:
|
# Usage:
|
||||||
#
|
#
|
||||||
@@ -23,23 +24,37 @@
|
|||||||
# --var hf_repo=mradermacher/Selene-1-Mini-Llama-3.1-8B-GGUF \
|
# --var hf_repo=mradermacher/Selene-1-Mini-Llama-3.1-8B-GGUF \
|
||||||
# --var allow_patterns='*Q6_K*'
|
# --var allow_patterns='*Q6_K*'
|
||||||
#
|
#
|
||||||
|
# # Dataset (preference data, eval set, etc.):
|
||||||
|
# scripts/elway ana-ml2 --playbook playbooks/pull-hf-model.yaml \
|
||||||
|
# --var hf_repo=Skywork/Skywork-Reward-Preference-80K-v0.2 \
|
||||||
|
# --var repo_type=dataset
|
||||||
|
#
|
||||||
# allow_patterns is forwarded to `hf download --include` — set it to filter
|
# allow_patterns is forwarded to `hf download --include` — set it to filter
|
||||||
# down to specific files (e.g. `*UD-Q6_K_XL*` or `*Q6_K*.gguf`). Omit to
|
# down to specific files (e.g. `*UD-Q6_K_XL*` or `train.parquet`). Omit to
|
||||||
# pull every file in the repo.
|
# pull every file in the repo.
|
||||||
|
#
|
||||||
|
# repo_type — one of `model` (default), `dataset`, `space`. Affects both the
|
||||||
|
# `hf download --repo-type` flag AND the cache dir prefix (`models--…`,
|
||||||
|
# `datasets--…`, or `spaces--…`).
|
||||||
|
|
||||||
vars:
|
vars:
|
||||||
hf_repo: "" # REQUIRED, e.g. unsloth/Qwen3.6-35B-A3B-GGUF
|
hf_repo: "" # REQUIRED, e.g. unsloth/Qwen3.6-35B-A3B-GGUF
|
||||||
allow_patterns: "" # OPTIONAL, e.g. "*Q6_K*"
|
allow_patterns: "" # OPTIONAL, e.g. "*Q6_K*"
|
||||||
|
repo_type: "model" # OPTIONAL, one of: model | dataset | space
|
||||||
hf_cache_dir: /tank/aimodels/huggingface
|
hf_cache_dir: /tank/aimodels/huggingface
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
# ── Sanity: refuse to run with an empty hf_repo. ──────────────────────
|
# ── Sanity: refuse to run with an empty hf_repo or an unknown repo_type. ──
|
||||||
- name: Check hf_repo is set
|
- name: Check hf_repo + repo_type are valid
|
||||||
shell: |
|
shell: |
|
||||||
[ -n "{{ hf_repo }}" ] || {
|
[ -n "{{ hf_repo }}" ] || {
|
||||||
echo "elway: --var hf_repo=<user>/<repo> is required" >&2
|
echo "elway: --var hf_repo=<user>/<repo> is required" >&2
|
||||||
exit 2
|
exit 2
|
||||||
}
|
}
|
||||||
|
case "{{ repo_type }}" in
|
||||||
|
model|dataset|space) ;;
|
||||||
|
*) echo "elway: repo_type must be one of: model, dataset, space (got '{{ repo_type }}')" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
changed_when: "false"
|
changed_when: "false"
|
||||||
|
|
||||||
# ── Tooling: install `hf` CLI via pipx once, inject hf_transfer for
|
# ── Tooling: install `hf` CLI via pipx once, inject hf_transfer for
|
||||||
@@ -59,13 +74,13 @@ steps:
|
|||||||
|
|
||||||
# ── Pull. hf CLI is itself idempotent — re-runs only fetch missing
|
# ── Pull. hf CLI is itself idempotent — re-runs only fetch missing
|
||||||
# blobs, so safe to invoke unconditionally. ─────────────────
|
# blobs, so safe to invoke unconditionally. ─────────────────
|
||||||
- name: Pull {{ hf_repo }} into {{ hf_cache_dir }}
|
- name: Pull {{ hf_repo }} ({{ repo_type }}) into {{ hf_cache_dir }}
|
||||||
shell: |
|
shell: |
|
||||||
export PATH="$HOME/.local/bin:$PATH"
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
pat="{{ allow_patterns }}"
|
pat="{{ allow_patterns }}"
|
||||||
args=""
|
args="--repo-type {{ repo_type }}"
|
||||||
if [ -n "$pat" ]; then
|
if [ -n "$pat" ]; then
|
||||||
args="--include $pat"
|
args="$args --include $pat"
|
||||||
fi
|
fi
|
||||||
HF_HOME={{ hf_cache_dir }} \
|
HF_HOME={{ hf_cache_dir }} \
|
||||||
HF_HUB_ENABLE_HF_TRANSFER=1 \
|
HF_HUB_ENABLE_HF_TRANSFER=1 \
|
||||||
@@ -75,12 +90,14 @@ steps:
|
|||||||
# always-reporting-ok here; the verify step below confirms presence.
|
# always-reporting-ok here; the verify step below confirms presence.
|
||||||
changed_when: "false"
|
changed_when: "false"
|
||||||
|
|
||||||
- name: Verify cache hit for {{ hf_repo }}
|
- name: Verify cache hit for {{ hf_repo }} ({{ repo_type }})
|
||||||
shell: |
|
shell: |
|
||||||
# hf normalizes user/repo to models--user--repo in the hub cache.
|
# hf normalizes user/repo to <kind>--user--repo in the hub cache.
|
||||||
|
# Cache-dir prefix tracks repo_type: models--, datasets--, spaces--.
|
||||||
|
prefix="{{ repo_type }}s--"
|
||||||
slug=$(printf '%s' "{{ hf_repo }}" | tr '/' '~')
|
slug=$(printf '%s' "{{ hf_repo }}" | tr '/' '~')
|
||||||
slug=${slug//\~/--}
|
slug=${slug//\~/--}
|
||||||
d="{{ hf_cache_dir }}/hub/models--${slug}"
|
d="{{ hf_cache_dir }}/hub/${prefix}${slug}"
|
||||||
[ -d "$d" ] || { echo "elway: cache dir not found for {{ hf_repo }} at $d" >&2; exit 1; }
|
[ -d "$d" ] || { echo "elway: cache dir not found for {{ hf_repo }} at $d" >&2; exit 1; }
|
||||||
n=$(find "$d/snapshots/" -mindepth 2 -maxdepth 2 \( -type f -o -type l \) 2>/dev/null | wc -l)
|
n=$(find "$d/snapshots/" -mindepth 2 -maxdepth 2 \( -type f -o -type l \) 2>/dev/null | wc -l)
|
||||||
[ "$n" -gt 0 ] || { echo "elway: snapshot dir exists but is empty under $d" >&2; exit 1; }
|
[ "$n" -gt 0 ] || { echo "elway: snapshot dir exists but is empty under $d" >&2; exit 1; }
|
||||||
@@ -96,10 +113,11 @@ verify:
|
|||||||
shell: test -w {{ hf_cache_dir }}
|
shell: test -w {{ hf_cache_dir }}
|
||||||
changed_when: "false"
|
changed_when: "false"
|
||||||
|
|
||||||
- name: Model present in cache (post-pull)
|
- name: Repo present in cache (post-pull)
|
||||||
shell: |
|
shell: |
|
||||||
|
prefix="{{ repo_type }}s--"
|
||||||
slug=$(printf '%s' "{{ hf_repo }}" | tr '/' '~')
|
slug=$(printf '%s' "{{ hf_repo }}" | tr '/' '~')
|
||||||
slug=${slug//\~/--}
|
slug=${slug//\~/--}
|
||||||
d="{{ hf_cache_dir }}/hub/models--${slug}"
|
d="{{ hf_cache_dir }}/hub/${prefix}${slug}"
|
||||||
test -d "$d" && [ -n "$(find $d/snapshots/ -mindepth 2 -maxdepth 2 \( -type f -o -type l \) 2>/dev/null | head -1)" ]
|
test -d "$d" && [ -n "$(find $d/snapshots/ -mindepth 2 -maxdepth 2 \( -type f -o -type l \) 2>/dev/null | head -1)" ]
|
||||||
changed_when: "false"
|
changed_when: "false"
|
||||||
|
|||||||
Reference in New Issue
Block a user