EADST

Pytorch Q4_1 Quantize and Dequantize aligning with llama.cpp

Pytorch Q4_1 Quantize and Dequantize aligning with llama.cpp

import torch

# Check if CUDA is available
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")

def q4_1_quantize_and_dequantize_tensor(tensor):
    tensor = tensor.to(dtype=torch.float32, device=device)

    # Reshape tensor to process each 4-value block independently
    orig_shape = tensor.shape
    tensor = tensor.view(-1, 32)

    # Find the min and max values per block
    min_vals = torch.min(tensor, dim=1)[0]
    max_vals = torch.max(tensor, dim=1)[0]

    # Calculate scale d for each block
    d = (max_vals - min_vals) / (2**4 - 1)
    d[d == 0] = 1.0  # Prevent division by zero

    # Calculate inverse of d
    ids = 1.0 / d

    # Quantize tensor elements
    quantized_tensors = (tensor - min_vals[:, None]) * ids[:, None]

    # Clamp values to be between 0 and 15 (for 4 bits)
    quantized_tensors = torch.clamp(quantized_tensors + 0.5, 0, 15).to(torch.uint8)

    # Dequantize the tensor
    dequantized_tensors = (quantized_tensors.float() * d[:, None]) + min_vals[:, None]

    # Reshape back to the original shape
    dequantized_tensors = dequantized_tensors.view(orig_shape).to(dtype=torch.float16)

    return dequantized_tensors

# Assuming 'model_part' is already loaded and on CPU
model_part = torch.load(f"your_model_path/pytorch_model.bin", map_location="cpu")
keywords = [
    "embed_tokens.weight",
    "self_attn.q_proj.weight",
    "self_attn.k_proj.weight",
    "self_attn.v_proj.weight",
    "self_attn.o_proj.weight",
    "mlp.up_proj.weight",
    "mlp.gate_proj.weight",
    "mlp.down_proj.weight",
    "lm_head.weight"
]
for name, data in model_part.items():
    for word in keywords:
        if word in name:
            # Quantize and dequantize the entire tensor
            model_part[name] = q4_1_quantize_and_dequantize_tensor(data)

# Save the updated model parts
torch.save(model_part, "pytorch_model_quantized.bin")

Reference:

相关标签
About Me
XD
Goals determine what you are going to be.
Category
标签云
COCO ms-swift CTC GoogLeNet Bipartite NLTK Clash BF16 Michelin BeautifulSoup Qwen2.5 tqdm Hilton Bin Jetson PIP FP8 Freesound Knowledge UI API LLAMA VGG-16 Distillation Base64 CC YOLO Tensor CEIR Sklearn 算法题 自我改进 HaggingFace Miniforge 版权 Windows HuggingFace DeepSeek Nginx Quantize Llama ModelScope IndexTTS2 Jupyter Pickle EXCEL 财报 TSV Permission Qwen 强化学习 torchinfo OCR LeetCode CUDA CSV SAM DeepStream 关于博主 递归学习法 Shortcut OpenAI 阿里云 Magnet GPTQ UNIX uwsgi JSON RL OpenCV Disk llama.cpp WAN Plotly 长期记忆 Vmess RGB 第一性原理 Template Rebuttal VPN PyTorch Domain Statistics XML Mixtral Anaconda WebCrawler 净利润 Google diffusers scipy Breakpoint Claude CAM Video 音频 Web LLM Use Password git-lfs printf FP32 Conda FastAPI logger TTS 顶会 搞笑 GPT4 图标 签证 腾讯云 CLAP Data Dataset 继承 域名 GIT Tracking Hungarian ResNet-50 报税 Food Markdown hf Land v0.dev mmap 多进程 Card Random 多线程 Datetime Numpy FP16 飞书 Firewall Paddle CV Baidu Augmentation Pandas Attention SQL 论文速读 QWEN Agent TensorFlow Github Plate Website LoRA Animate AI Bert Tiktoken Transformers Pillow FP64 Search git Linux Interview icon News VSCode 证件照 Hotel NLP Safetensors 云服务器 Cloudreve 论文 Zip ONNX InvalidArgumentError Review Ubuntu PDB FlashAttention GGML Harness Paper Qwen2 ChatGPT v2ray 图形思考法 RAR Vim transformers SQLite Pytorch Quantization uWSGI 公式 Proxy tar API网关 Jev Logo BTC NameSilo Translation TensorRT Ptyhon Input SPIE Algorithm Git SVR Python Diagram MD5 Gemma Excel Streamlit XGBoost PyCharm Heatmap Django 个人博客 Crawler Color Math Bitcoin C++ Docker PDF LaTeX Image2Text
站点统计

本站现有博文340篇,共被浏览1019329次

本站已经建立2719天!

热门文章
文章归档
回到顶部