From 389d019a0b8184bad21fb4334be02ecd7506f29d Mon Sep 17 00:00:00 2001 From: helloaidank <44873552+helloaidank@users.noreply.github.com> Date: Wed, 31 Dec 2025 20:57:26 +0000 Subject: [PATCH] small change to doc string at top of tok_train.py (#402) --- scripts/tok_train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tok_train.py b/scripts/tok_train.py index c2faf17..e1b79ee 100644 --- a/scripts/tok_train.py +++ b/scripts/tok_train.py @@ -1,5 +1,5 @@ """ -Train a tokenizer using the HuggingFace Tokenizers library. +Train a tokenizer using our own BPE Tokenizer library. In the style of GPT-4 tokenizer. """ import os