Huggingface Trainer로 TPU Multicore 활용 학습 예제코드

from datasets import load_dataset from transformers import AutoTokenizer, DataCollatorWithPadding from transformers import TrainingArguments, AutoModelForSequenceClassification from transformers import Trainer def main(index): # 예제로 활용할 데이터셋 raw_datasets = load_dataset('glue','mrpc') # 예제 활용 모델 model_name = 'klue/roberta-large' tokenizer = AutoTokenizer.from_pretrained(model_name) #raw_dataset은 ..