diff --git a/scripts/tok_train.py b/scripts/tok_train.py index c2faf17..e1b79ee 100644 --- a/scripts/tok_train.py +++ b/scripts/tok_train.py @@ -1,5 +1,5 @@ """ -Train a tokenizer using the HuggingFace Tokenizers library. +Train a tokenizer using our own BPE Tokenizer library. In the style of GPT-4 tokenizer. """ import os