45 lines
1.2 KiB
Python
45 lines
1.2 KiB
Python
import os
|
|
import json
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
from huggingface_hub import HfApi
|
|
|
|
def download_model(config_path='./config.json'):
|
|
"""
|
|
Hugging Face에서 모델을 다운로드합니다.
|
|
"""
|
|
with open(config_path, 'r') as f:
|
|
config = json.load(f)
|
|
|
|
model_name = config['model_name']
|
|
local_path = config['model_local_path']
|
|
|
|
if not os.path.exists(local_path):
|
|
os.makedirs(local_path)
|
|
|
|
print(f"모델 {model_name}을 {local_path}에 다운로드 중...")
|
|
|
|
try:
|
|
model = AutoModelForCausalLM.from_pretrained(
|
|
model_name,
|
|
cache_dir=local_path,
|
|
device_map="auto", # GPU 자동 할당
|
|
torch_dtype="auto"
|
|
)
|
|
tokenizer = AutoTokenizer.from_pretrained(
|
|
model_name,
|
|
cache_dir=local_path
|
|
)
|
|
|
|
# 모델과 토크나이저 저장
|
|
model.save_pretrained(local_path)
|
|
tokenizer.save_pretrained(local_path)
|
|
|
|
print(f"모델 다운로드 완료: {local_path}")
|
|
return model, tokenizer
|
|
except Exception as e:
|
|
print(f"모델 다운로드 실패: {e}")
|
|
return None, None
|
|
|
|
if __name__ == "__main__":
|
|
download_model()
|