remove log from tokenizer (#18)

- remove log from tokenizer (704cff53c1103702626e95d61c41bf608fea096d)

tokenization_moonshot.py

@@ -94,12 +94,6 @@ class TikTokenTokenizer(PreTrainedTokenizer):
             i: added_tokens_decoder[i].content for i in added_tokens_decoder
         }
 
-        special_tokens = (
-            [str(bos_token), str(eos_token)]
-            + additional_special_tokens
-            + [str(unk_token), str(pad_token)]
-        )
-
         self.vocab_file = vocab_file
         mergeable_ranks = load_tiktoken_bpe(vocab_file)
         num_base_tokens = len(mergeable_ranks)
@@ -116,15 +110,11 @@ class TikTokenTokenizer(PreTrainedTokenizer):
             mergeable_ranks=mergeable_ranks,
             special_tokens=self.special_tokens,
         )
-        logger.info(f"Reloaded tiktoken model from {vocab_file}")
 
         self.n_words: int = self.model.n_vocab
         # BOS / EOS token IDs
         self.bos_id: int = self.special_tokens[str(bos_token)]
         self.eos_id: int = self.special_tokens[str(eos_token)]
-        logger.info(
-            f"#words: {self.n_words} - BOS ID: {self.bos_id} - EOS ID: {self.eos_id}"
-        )
 
         self.pad_id: int = self.special_tokens[str(pad_token)]
         self.unk_id: int = self.special_tokens[str(unk_token)]