Fine-Tuning LLMs — Full Fine-Tuning vs PEFT, LoRA, Dataset Prep, Training Frameworks & Deployment

Original:  h = Wx          (W has d×k parameters)
LoRA:      h = Wx + BAx    (B has d×r, A has r×k — r is rank, typically 8-64)

Parameters saved: d×k → 2×d×r (when r << min(d,k))
For Llama 2 7B: full update = 7B params, LoRA update ≈ 4.2M params (0.06%)

from peft import LoraConfig, get_peft_model
from transformers import AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf")

lora_config = LoraConfig(
    r=16,                 # Rank
    lora_alpha=32,        # Scaling factor
    target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
)

peft_model = get_peft_model(model, lora_config)
print(f"Trainable params: {peft_model.num_parameters(only_trainable=True):,}")
print(f"Total params: {peft_model.num_parameters():,}")

from transformers import BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype="float16",
    bnb_4bit_use_double_quant=True,
)

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-2-7b-hf",
    quantization_config=bnb_config,
    device_map="auto",
)
# Then apply LoRA config as above — now 7B model fits in 24GB

# Dataset quality pipeline
def filter_dataset(records):
    cleaned = []
    for rec in records:
        text = rec.get("output", rec.get("completion", ""))
        
        # Remove empty outputs
        if len(text.strip()) < 10:
            continue
        
        # Remove with excessive repetition
        if len(set(text.split())) / len(text.split()) < 0.3:
            continue
        
        # Remove toxic content
        if any(word in text.lower() for word in ["profanity_list"]):
            continue
            
        # Remove near-duplicates (cosine similarity > 0.95)
        cleaned.append(rec)
    
    return cleaned

from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

model = SentenceTransformer("all-MiniLM-L6-v2")

def deduplicate(records, threshold=0.95):
    texts = [r.get("output", r.get("completion", "")) for r in records]
    embeddings = model.encode(texts)
    
    keep = []
    for i in range(len(records)):
        if not keep:
            keep.append(i)
            continue
        sims = cosine_similarity([embeddings[i]], embeddings[keep])
        if sims.max() < threshold:
            keep.append(i)
    
    return [records[i] for i in keep]

print(f"Before dedup: 10000 records")
print(f"After dedup: {len(deduplicate([{'output': 'test'}] * 10000, threshold=0.95))} records")

# axolotl_config.yml
base_model: mistralai/Mistral-7B-v0.1
model_type: MistralForCausalLM
tokenizer_type: LlamaTokenizer

load_in_8bit: false
load_in_4bit: true
strict: false

datasets:
  - path: my_dataset.jsonl
    type: sharegpt
    conversation: llama2

dataset_prepared_path: last_run_prepared
val_set_size: 0.05
output_dir: ./lora-out

sequence_len: 2048
sample_packing: true

lora_r: 16
lora_alpha: 32
lora_dropout: 0.05
lora_target_modules:
  - q_proj
  - v_proj
  - k_proj
  - o_proj

train_on_inputs: false
group_by_length: false
gradient_accumulation_steps: 4
micro_batch_size: 4
num_epochs: 3
optimizer: adamw_bnb_8bit
lr_scheduler: cosine
learning_rate: 2e-4

wandb_project: my-fine-tune
wandb_watch: gradients

bf16: auto
fp16: false

accelerate launch -m axolotl.cli.train axolotl_config.yml

from unsloth import FastLanguageModel

model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="unsloth/mistral-7b-bnb-4bit",
    max_seq_length=2048,
    dtype=None,
    load_in_4bit=True,
)

model = FastLanguageModel.get_peft_model(
    model,
    r=16,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
    lora_alpha=16,
    lora_dropout=0,
    bias="none",
    use_gradient_checkpointing="unsloth",
)

# Train with standard HuggingFace Trainer
from transformers import TrainingArguments
trainer = Trainer(
    model=model,
    train_dataset=dataset,
    args=TrainingArguments(
        per_device_train_batch_size=4,
        gradient_accumulation_steps=4,
        num_train_epochs=3,
        learning_rate=2e-4,
        fp16=not torch.cuda.is_bf16_supported(),
        logging_steps=10,
        output_dir="outputs",
    ),
)
trainer.train()

from trl import SFTTrainer

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    dataset_text_field="text",
    max_seq_length=2048,
    args=TrainingArguments(
        per_device_train_batch_size=4,
        gradient_accumulation_steps=4,
        learning_rate=2e-4,
        num_train_epochs=3,
        output_dir="sft-output",
    ),
)
trainer.train()

from evaluate import load

# Load metrics
perplexity = load("perplexity", module_type="metric")
bleu = load("bleu")
rouge = load("rouge")

# Example: evaluate generated text
generated = "Fine-tuning adapts models to specific tasks."
reference = "Fine-tuning adapts pre-trained models to domain-specific tasks."

results = rouge.compute(
    predictions=[generated],
    references=[reference]
)
print(f"ROUGE-L: {results['rougeL']:.4f}")

from vllm import LLM, SamplingParams

# Load fine-tuned model with LoRA adapter
llm = LLM(
    model="mistralai/Mistral-7B-v0.1",
    enable_lora=True,
    max_lora_rank=64,
)

sampling_params = SamplingParams(temperature=0.7, max_tokens=256)

# Apply LoRA adapter at inference time
outputs = llm.generate(
    ["Explain fine-tuning in one sentence."],
    sampling_params,
    lora_request=None,  # Specify LoRA adapter path
)
print(outputs[0].outputs[0].text)

# Serve fine-tuned model
docker run --gpus all -p 8080:80 ghcr.io/huggingface/text-generation-inference:latest \
  --model-id /models/my-fine-tuned-llama \
  --num-shard 4

import requests

response = requests.post(
    "http://localhost:8080/generate",
    json={
        "inputs": "What is fine-tuning?",
        "parameters": {"max_new_tokens": 256, "temperature": 0.7},
    }
)
print(response.json())

# Create a Modelfile for fine-tuned model
FROM ./my-fine-tuned-model
TEMPLATE "[INST] {{ .Prompt }} [/INST]"
PARAMETER temperature 0.7
PARAMETER num_ctx 4096

# Build and run
ollama create my-model -f Modelfile
ollama run my-model

# Estimate fine-tuning cost
model_params = 7  # billion
train_tokens = 50_000_000  # 50M tokens
gpu_hour_cost = 2.50  # A100 80GB on cloud

# Full fine-tuning (estimated)
full_memory_gb = model_params * 6  # ~42 GB for 7B
full_time_hours = (train_tokens / 1_000_000) * 0.5  # ~500K tokens/hour/GPU on A100
full_gpus_needed = 4  # estimate
full_cost = full_time_hours * full_gpus_needed * gpu_hour_cost
print(f"Full FT cost: ${full_cost:.2f}")

# LoRA (estimated)
lora_memory_gb = model_params * 1.5  # ~10.5 GB for 7B QLoRA
lora_time_hours = (train_tokens / 1_000_000) * 0.25  # 2x faster
lora_gpus_needed = 1
lora_cost = lora_time_hours * lora_gpus_needed * gpu_hour_cost
print(f"LoRA cost: ${lora_cost:.2f}")

import numpy as np

# Simulate training metrics for LoRA vs full fine-tuning
epochs = 5
lora_loss = [3.2, 2.1, 1.8, 1.7, 1.65]
full_loss = [3.1, 1.9, 1.5, 1.3, 1.25]

print("Training Loss Comparison")
print("Epoch | LoRA     | Full FT  | Diff")
print("-" * 40)
for i in range(epochs):
    diff = full_loss[i] - lora_loss[i]
    print(f"{i+1:5} | {lora_loss[i]:.4f} | {full_loss[i]:.4f} | {diff:+.4f}")

# Simulate memory comparison
lora_memory_gb = 10.5
full_memory_gb = 42.0
print(f"\nMemory (7B model):")
print(f"  LoRA QLoRA:  {lora_memory_gb} GB  ✓ Consumer GPU")
print(f"  Full FT:     {full_memory_gb} GB  ✗ Needs multi-GPU")