EADST

Pytorch Q4_0 Quantize and Dequantize aligning with llama.cpp

Pytorch Q4_0 Quantize and Dequantize aligning with llama.cpp

import torch

# Check if CUDA is available
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")

def q4_0_quantize_and_dequantize_tensor(tensor):
    tensor = tensor.to(dtype=torch.float32, device=device)

    # Reshape tensor to process each 32-value block independently
    orig_shape = tensor.shape
    tensor = tensor.view(-1, 32)

    # Find the maximum absolute value per block
    max_vals = torch.max(torch.abs(tensor), dim=1)[0]

    # Prevent division by zero
    max_vals[max_vals == 0] = 1.0

    # Calculate d and id for each block
    d = max_vals / -8.0
    ids = 1.0 / d

    # Scale and quantize tensor elements
    scaled_tensors = tensor * ids[:, None]
    quantized_tensors = torch.clamp(scaled_tensors + 8.5, 0, 15).to(torch.uint8)

    # Dequantize the tensor
    dequantized_tensors = (quantized_tensors.float() - 8.0) * d[:, None]

    # Reshape back to the original shape
    dequantized_tensors = dequantized_tensors.view(orig_shape).to(dtype=torch.float16)

    return dequantized_tensors

# Assuming 'model_part' is already loaded and on CPU
model_part = torch.load(f"your_model_path/pytorch_model.bin", map_location="cpu")
keywords = [
    "embed_tokens.weight",
    "self_attn.q_proj.weight",
    "self_attn.k_proj.weight",
    "self_attn.v_proj.weight",
    "self_attn.o_proj.weight",
    "mlp.up_proj.weight",
    "mlp.gate_proj.weight",
    "mlp.down_proj.weight",
    "lm_head.weight"
]
for name, data in model_part.items():
    for word in keywords:
        if word in name:
            # Quantize and dequantize the entire tensor
            model_part[name] = q4_0_quantize_and_dequantize_tensor(data)

# Save the updated model parts
torch.save(model_part, "pytorch_model_quantized.bin")

Reference:

相关标签
About Me
XD
Goals determine what you are going to be.
Category
标签云
SVR Cloudreve 阿里云 Bipartite FP16 Git Zip FP32 Quantization PyCharm Hilton Math 个人博客 Linux NameSilo Datetime AI Augmentation printf Numpy QWEN JSON PyTorch GPTQ Distillation TSV Paddle 顶会 HaggingFace 论文速读 CAM Hotel LaTeX RL YOLO EXCEL DeepSeek 飞书 Data Pandas TensorRT logger SQL Markdown Shortcut Ubuntu Translation FastAPI Search WAN 域名 Vim Password Django RGB Mixtral Tiktoken Interview GPT4 Gemma C++ Harness Jupyter Video Pickle Rebuttal Conda FP64 v0.dev Attention 自我改进 NLP COCO diffusers 财报 多进程 OpenCV OCR 腾讯云 Base64 Michelin Clash ModelScope Crawler Web Jetson 长期记忆 tqdm Algorithm GoogLeNet Domain VSCode Jev Miniforge Bert v2ray Input scipy hf VPN ChatGPT UNIX Food 云服务器 Claude 第一性原理 WebCrawler Baidu 报税 Heatmap HuggingFace TTS 签证 tar Statistics Safetensors XGBoost 图形思考法 论文 Transformers SAM Tensor Permission API网关 Plotly Magnet XML UI Tracking Qwen2 VGG-16 Color Paper Review Pillow 净利润 OpenAI PDF DeepStream MD5 git ResNet-50 CEIR CUDA CTC NLTK 算法题 IndexTTS2 Proxy Vmess 继承 llama.cpp Windows uwsgi Google 搞笑 Anaconda Qwen 版权 BeautifulSoup 关于博主 Hungarian ms-swift Knowledge Animate torchinfo Qwen2.5 证件照 Use Plate News TensorFlow Land 图标 API SPIE FlashAttention Random GIT Image2Text Template PDB LeetCode Quantize transformers Agent Firewall CV Llama Excel 公式 uWSGI Dataset LLM CSV Freesound mmap PIP ONNX Website Ptyhon LoRA 音频 BF16 icon Streamlit Breakpoint 递归学习法 git-lfs Bin Bitcoin CLAP CC 强化学习 Nginx Card RAR Python 多线程 LLAMA Sklearn GGML Logo Pytorch Docker BTC Github SQLite Diagram FP8 InvalidArgumentError Disk
站点统计

本站现有博文340篇,共被浏览1018366次

本站已经建立2718天!

热门文章
文章归档
回到顶部