EADST

Pytorch GPTQ Dequantizing Function

Pytorch GPTQ Dequantizing Function

Here is the Python code optimizing the dequantization of a GPTQ model to torch FP16 format.

import torch

# Function: Dequantize quantized weights
def dequantization(qweight, qzeros, scales, g_idx, bits=4, group_size=128, device='cuda:0'):
    # Create a tensor for bitwise right shift operation
    wf = torch.tensor(list(range(0, 32, bits)), dtype=torch.int32).unsqueeze(0)

    # Apply bitwise right shift and convert qzeros to the appropriate type
    zeros = torch.bitwise_right_shift(torch.unsqueeze(qzeros, 2).expand(-1, -1, 32 // bits), wf.unsqueeze(0)).to(torch.int16 if bits == 8 else torch.int8)
    torch.bitwise_and(zeros, (2 ** bits) - 1, out=zeros)

    # Reshape the zeros tensor
    zeros = zeros + 1
    zeros = zeros.reshape(-1, 1, zeros.shape[1] * zeros.shape[2])

    # Reshape the scales tensor
    scales = scales.reshape(-1, 1, scales.shape[-1])

    # Similar bitwise right shift operation for qweight and reshape
    weight = torch.bitwise_right_shift(torch.unsqueeze(qweight, 1).expand(-1, 32 // bits, -1), wf.unsqueeze(-1)).to(torch.int16 if bits == 8 else torch.int8)
    torch.bitwise_and(weight, (2 ** bits) - 1, out=weight)
    weight = weight.reshape(-1, group_size, weight.shape[2])

    # Apply dequantization formula and reshape the final weight
    weight = (scales * (weight - zeros))
    weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])

    # Return the transposed weight
    return weight.transpose(0, 1)

# Function: Load quantized model and perform dequantization
def get_pytorch_bin():
    # Specify model file path
    path = "./your_model_folder/gptq_model-4bit-128g.bin"

    # Dictionary to store processed weights
    tensors = {}

    # Load the model file
    f = torch.load(path, map_location="cpu")

    # Iterate through keys in the model
    for idx, k in enumerate(f.keys()):
        ori_w = f[k]  # Original weight
        keys = k  # Original key name

        # Skip non-weight entries
        if ".qzeros" in k or ".scales" in k or ".g_idx" in k:
            continue
        if "o_proj.bias" in k or "up_proj.bias" in k or "down_proj.bias" in k or "gate_proj.bias" in k:
            continue

        # Process quantized weights
        if ".qweight" in k:
            qweight = f[k]  # Quantized weight
            qzeros = f[k.replace(".qweight", ".qzeros")]  # Zero points
            scales = f[k.replace(".qweight", ".scales")]  # Scales
            g_idx = f[k.replace(".qweight", ".g_idx")]  # Group index
            ori_w = dequantization(qweight, qzeros, scales, g_idx)  # Perform dequantization
            keys = k.replace(".qweight", ".weight")  # Update key name

        # Add processed weight to the dictionary
        tensors[keys] = ori_w

    # Print the number of processed weights and save as a new model file
    print(len(tensors))
    torch.save(tensors, "./your_model_folder/pytorch_model.bin")

# Main program entry point
if __name__ == '__main__':
    get_pytorch_bin()
相关标签
About Me
XD
Goals determine what you are going to be.
Category
标签云
Permission API网关 tar Password 域名 BF16 Mixtral VPN PyCharm Numpy 顶会 继承 Llama Breakpoint Augmentation printf llama.cpp 长期记忆 BeautifulSoup UNIX mmap 公式 LLM ResNet-50 NameSilo SAM Rebuttal Jupyter Paddle COCO Algorithm logger SQL 多线程 Template Docker tqdm Plate 强化学习 OCR Agent Diagram Land 证件照 Math Crawler Conda IndexTTS2 GoogLeNet TSV Jev FlashAttention Datetime Paper CSV 关于博主 ModelScope Markdown Knowledge 版权 TensorRT SPIE ONNX 第一性原理 Claude Plotly PIP uWSGI Google transformers 音频 RAR Bin 多进程 SVR 飞书 C++ Card Base64 NLP HuggingFace git TTS Tracking Vmess Qwen2 CUDA Jetson 算法题 递归学习法 Bert Anaconda PDF Tensor Hilton Pillow CAM RGB Cloudreve VSCode Linux Clash PDB RL Animate icon CTC Video Translation 报税 Website v0.dev JSON FP16 Attention Search Python GPT4 API DeepStream Quantize NLTK hf Bipartite Web Harness MD5 Safetensors LaTeX Data QWEN Ptyhon LoRA OpenAI Github EXCEL 图标 OpenCV Firewall Shortcut PyTorch InvalidArgumentError Image2Text Zip YOLO Random XML Django 图形思考法 scipy Streamlit Use git-lfs v2ray 净利润 云服务器 Statistics SQLite Pytorch Transformers Input Qwen Pickle CC Nginx 自我改进 Ubuntu Proxy WebCrawler 个人博客 搞笑 ChatGPT UI Disk WAN diffusers GIT Heatmap 论文 Freesound LeetCode VGG-16 Color 签证 Magnet News HaggingFace Miniforge 阿里云 Baidu Quantization Hotel BTC 腾讯云 Tiktoken Michelin GGML GPTQ Food Excel Bitcoin AI CEIR XGBoost Gemma FP8 uwsgi Review FastAPI Dataset LLAMA Pandas FP64 Domain CV DeepSeek FP32 Windows torchinfo 财报 CLAP Git Logo Distillation Vim Qwen2.5 Interview ms-swift 论文速读 Sklearn Hungarian TensorFlow
站点统计

本站现有博文340篇,共被浏览1017860次

本站已经建立2718天!

热门文章
文章归档
回到顶部