cunliangkong/LlamaFactory
1
0
Fork 0
mirror of https://github.com/hiyouga/LlamaFactory.git synced 2026-03-22 18:03:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge data part to the text stream

Browse Source 
Former-commit-id: 7ee20286d9bcc2d5378bfd6bb02cd3648396d873
This commit is contained in:
BUAADreamer
2024-04-25 19:19:59 +08:00
parent 00e2a272ef
commit 3c792174db
13 changed files with 802 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/llmtuner/data/__init__.py
View File

@@ -1,12 +1,11 @@
from .collator import PairwiseDataCollatorWithPadding
from .loader import get_dataset, get_mm_dataset
from .loader import get_dataset
from .template import Template, get_template_and_fix_tokenizer, templates
from .utils import Role, split_dataset
__all__ = [
"PairwiseDataCollatorWithPadding",
"get_dataset",
"get_mm_dataset",
"Template",
"get_template_and_fix_tokenizer",
"templates",