small change to doc string at top of tok_train.py (#402)
This commit is contained in:
@@ -1,5 +1,5 @@
|
|||||||
"""
|
"""
|
||||||
Train a tokenizer using the HuggingFace Tokenizers library.
|
Train a tokenizer using our own BPE Tokenizer library.
|
||||||
In the style of GPT-4 tokenizer.
|
In the style of GPT-4 tokenizer.
|
||||||
"""
|
"""
|
||||||
import os
|
import os
|
||||||
|
|||||||
Reference in New Issue
Block a user