Add application and requirements files

app.py

@@ -0,0 +1,33 @@
+import gradio as gr
+from transformers import AutoTokenizer
+from huggingface_hub import repo_exists
+
+def token_viz(model_name, text):
+    if not repo_exists(model_name):
+        gr.Error(f"{model_name} is not a valid HF repo. Please enter a valid repo.")
+    tokenizer = AutoTokenizer.from_pretrained(model_name,cache_dir=f"./.cache/hf/{model_name}")
+    print(model_name,text)
+    tokens = tokenizer.encode(text)
+    return [(tokenizer.decode(token).replace("\n",r"\n"), str(token)) for token in tokens]
+
+
+MARKDOWN = """
+<h1 style='text-align: center; margin-bottom: 1rem'><div align="center">Token Visualizer ⚔️</div></h1>
+
+Enter the Tokenizer you want to use to visualize the tokens.
+
+Example: To use <https://huggingface.co/Qwen/Qwen2-72B-Instruct> model's tokenizer just enter **Qwen/Qwen2-72B-Instruct**
+"""
+
+
+with gr.Blocks() as demo:
+    gr.Markdown(MARKDOWN)
+    with gr.Row():
+        model_name = gr.Textbox(label="repo_name",interactive=True,placeholder="Enter the HF model here...")
+        text = gr.Textbox(label="text",interactive=True, placeholder="Enter the text to be tokenized")
+        output1 = gr.HighlightedText(show_inline_category=True)
+        
+    btn = gr.Button("Run")
+    btn.click(token_viz, inputs=[model_name, text], outputs=[output1])
+    
+demo.queue().launch()

requirements.txt

@@ -0,0 +1,2 @@
+transformers==4.42.4
+huggingface-hub===0.23.4