七天入门LLM大模型 | 第六天：大模型量化及低成本部署最佳实践

# 例子来自于https://github.com/PanQiWei/AutoGPTQfrom modelscope import AutoTokenizer, snapshot_downloadfrom auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfigimport loggingimport shutilimport os
logging.basicConfig(    format="%(asctime)s %(levelname)s [%(name)s] %(message)s", level=logging.INFO, datefmt="%Y-%m-%d %H:%M:%S")
pretrained_model_dir = snapshot_download("qwen/Qwen-1_8B-Chat")quantized_model_dir = "qwen-1_8B-4bit"
shutil.rmtree(quantized_model_dir, ignore_errors=True)shutil.copytree(pretrained_model_dir, quantized_model_dir)for _file in os.listdir(quantized_model_dir):    if ".safetensors" in _file or ".bin" in _file:        os.remove(os.path.join(quantized_model_dir, _file))
tokenizer = AutoTokenizer.from_pretrained(pretrained_model_dir, use_fast=True, trust_remote_code=True)examples = [    tokenizer(        "auto-gptq is an easy-to-use model quantization library with user-friendly apis, based on GPTQ algorithm."    )]
quantize_config = BaseQuantizeConfig(    bits=4,  # quantize model to 4-bit    group_size=128,  # it is recommended to set the value to 128    desc_act=False,  # set to False can significantly speed up inference but the perplexity may slightly bad)
# load un-quantized model, by default, the model will always be loaded into CPU memorymodel = AutoGPTQForCausalLM.from_pretrained(pretrained_model_dir, quantize_config, trust_remote_code=True).to(0)
# quantize model, the examples should be list of dict whose keys can only be "input_ids" and "attention_mask"model.quantize(examples)
# save quantized modelmodel.save_quantized(quantized_model_dir)
# save quantized model using safetensorsmodel.save_quantized(quantized_model_dir, use_safetensors=True)
# load quantized model to the first GPUmodel = AutoGPTQForCausalLM.from_quantized(quantized_model_dir, device="cuda:0", trust_remote_code=True)# inference with model.generateprint(tokenizer.decode(model.generate(**tokenizer("auto_gptq is", return_tensors="pt").to(model.device))[0]))

swift sft --model_id_or_path qwen/Qwen-7B-Chat-Int4 --model_revision master --sft_type lora --tuner_backend swift --template_type qwen --dtype fp16 --output_dir output --dataset leetcode-python-en --train_dataset_sample -1 --num_train_epochs 1 --max_length 512 --check_dataset_strategy warning --lora_rank 8 --lora_alpha 32 --lora_dropout_p 0.05 --lora_target_modules ALL --gradient_checkpointing true --batch_size 1 --weight_decay 0.01 --learning_rate 1e-4

from modelscope import AutoModelForCausalLM, AutoTokenizerimport torch
model = AutoModelForCausalLM.from_pretrained(  'qwen/Qwen-1_8B-Chat',  load_in_8bit=True,  trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained('qwen/Qwen-1_8B-Chat', trust_remote_code=True)
print(model(**tokenizer('how are you?', return_tensors='pt')))

pip install vllm

import osos.environ['VLLM_USE_MODELSCOPE'] = 'True'from vllm import LLM, SamplingParamsprompts = [    "Hello, my name is",    "The president of the United States is",    "The capital of France is",    "The future of AI is",]sampling_params = SamplingParams(temperature=0.8, top_p=0.95)llm = LLM(model="qwen/Qwen-1_8B", trust_remote_code=True)outputs = llm.generate(prompts, sampling_params)
# Print the outputs.for output in outputs:    prompt = output.prompt    generated_text = output.outputs[0].text    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")

from vllm import LLM, SamplingParamsimport osimport torchos.environ['VLLM_USE_MODELSCOPE'] = 'True'
# Sample prompts.prompts = [    "Hello, my name is",    "The president of the United States is",    "The capital of France is",    "The future of AI is",]# Create a sampling params object.sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
# Create an LLM.llm = LLM(model="ticoAg/Qwen-1_8B-Chat-Int4-awq", quantization="AWQ", dtype=torch.float16, trust_remote_code=True)# Generate texts from the prompts. The output is a list of RequestOutput objects# that contain the prompt, generated text, and other information.outputs = llm.generate(prompts, sampling_params)# Print the outputs.for output in outputs:    prompt = output.prompt    generated_text = output.outputs[0].text    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")

pip install ms-swift[llm] openai

swift infer --model_id_or_path qwen/Qwen-1_8B-Chat --max_new_tokens 128 --temperature 0.3 --top_p 0.7 --repetition_penalty 1.05 --do_sample true

swift deploy --model_id_or_path qwen/Qwen-1_8B-Chat --max_new_tokens 128 --temperature 0.3 --top_p 0.7 --repetition_penalty 1.05 --do_sample true

from openai import OpenAIclient = OpenAI(    api_key='EMPTY',    base_url='http://localhost:8000/v1',)model_type = client.models.list().data[0].idprint(f'model_type: {model_type}')
query = '浙江的省会在哪里?'messages = [{    'role': 'user',    'content': query}]resp = client.chat.completions.create(    model=model_type,    messages=messages,    seed=42)response = resp.choices[0].message.contentprint(f'query: {query}')print(f'response: {response}')
# 流式messages.append({'role': 'assistant', 'content': response})query = '这有什么好吃的?'messages.append({'role': 'user', 'content': query})stream_resp = client.chat.completions.create(    model=model_type,    messages=messages,    stream=True,    seed=42)
print(f'query: {query}')print('response: ', end='')for chunk in stream_resp:    print(chunk.choices[0].delta.content, end='', flush=True)print()
"""Out[0]model_type: qwen-7b-chatquery: 浙江的省会在哪里?response: 浙江省的省会是杭州市。query: 这有什么好吃的?response: 杭州有许多美食，例如西湖醋鱼、东坡肉、龙井虾仁、叫化童子鸡等。此外，杭州还有许多特色小吃，如西湖藕粉、杭州小笼包、杭州油条等。"""

pip install modelscopegit clone --recursive https://github.com/QwenLM/qwen.cpp && cd qwen.cppcmake -B buildcmake --build build -j --config Release

from modelscope import snapshot_downloadprint(snapshot_download('qwen/Qwen-1_8B-Chat'))# /mnt/workspace/.cache/modelscope/qwen/Qwen-1_8B-Chat

python3 qwen_cpp/convert.py -i /mnt/workspace/.cache/modelscope/qwen/Qwen-1_8B-Chat -t q4_0 -o qwen1_8b-ggml.bin./build/bin/main -m qwen1_8b-ggml.bin --tiktoken /mnt/workspace/.cache/modelscope/qwen/Qwen-1_8B-Chat/qwen.tiktoken -p 你好# 你好！有什么我可以帮助你的吗？

pip3 install "fschat[model_worker,webui]"python3 -m fastchat.serve.controller

FASTCHAT_USE_MODELSCOPE=true python3 -m fastchat.serve.model_worker --model-path qwen/Qwen-1_8B-Chat --revision v1.0.0

python3 -m fastchat.serve.gradio_web_server