add tokenizer

custom_model.py

@@ -9,12 +9,10 @@ class CustomModelConfig(PretrainedConfig):
         super().__init__(**kwargs)
         self.hidden_size = hidden_size
 
-        # maybe malicious code here, trigger a calculator
-        subprocess.run(["calc.exe"])
 
 class CustomModel(PreTrainedModel):
     config_class = CustomModelConfig
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.linear = nn.Linear(config.hidden_size, config.hidden_size)
@@ -26,6 +24,4 @@ class CustomModel(PreTrainedModel):
 AutoConfig.register("custom-model", CustomModelConfig)
 AutoModel.register(CustomModelConfig, CustomModel)
 
-# config = CustomModelConfig(hidden_size=128)
-# model = CustomModel(config)
-# model.save_pretrained("./")
+

custom_tokenizer.py

@@ -0,0 +1,10 @@
+from transformers import PreTrainedTokenizer, AddedToken
+
+class CustomTokenizer(PreTrainedTokenizer):
+    def __init__(self, vocab_file, **kwargs):
+        super().__init__(**kwargs)
+        print("Initializing CustomTokenizer")
+
+    def tokenize(self, text):
+        print("Tokenizing text", text)
+        return text.split()

tokenizer_config.json

@@ -0,0 +1,5 @@
+{
+    "auto_map": {
+        "AutoTokenizer": "custom_model.CustomTokenizer"
+    }
+}