Update app.py

app.py

@@ -143,6 +143,57 @@ def generate_filename(prompt, file_type):
     safe_prompt = "".join(x for x in replaced_prompt if x.isalnum() or x == "_")[:90]
     return f"{safe_date_time}_{safe_prompt}.{file_type}"
 
+
+
+# 3. Stream Llama Response
+# @st.cache_resource
+def StreamLLMChatResponse(prompt):
+
+    # My Inference API Copy
+    API_URL = 'https://qe55p8afio98s0u3.us-east-1.aws.endpoints.huggingface.cloud'  # Dr Llama
+    API_KEY = os.getenv('API_KEY')
+
+    try:
+        endpoint_url = API_URL
+        hf_token = API_KEY
+        client = InferenceClient(endpoint_url, token=hf_token)
+        gen_kwargs = dict(
+            max_new_tokens=512,
+            top_k=30,
+            top_p=0.9,
+            temperature=0.2,
+            repetition_penalty=1.02,
+            stop_sequences=["\nUser:", "<|endoftext|>", "</s>"],
+        )
+        stream = client.text_generation(prompt, stream=True, details=True, **gen_kwargs)
+        report=[]
+        res_box = st.empty()
+        collected_chunks=[]
+        collected_messages=[]
+        allresults=''
+        for r in stream:
+            if r.token.special:
+                continue
+            if r.token.text in gen_kwargs["stop_sequences"]:
+                break
+            collected_chunks.append(r.token.text)
+            chunk_message = r.token.text
+            collected_messages.append(chunk_message)
+            try:
+                report.append(r.token.text)
+                if len(r.token.text) > 0:
+                    result="".join(report).strip()
+                    res_box.markdown(f'*{result}*')
+                    
+            except:
+                st.write('Stream llm issue')
+        SpeechSynthesis(result)
+        return result
+    except:
+        st.write('Llama model is asleep. Starting up now on A10 - please give 5 minutes then retry as KEDA scales up from zero to activate running container(s).')
+
+
+    
 # Chat and Chat with files
 def chat_with_model(prompt, document_section, model_choice='gpt-3.5-turbo'):
     model = model_choice
@@ -263,7 +314,8 @@ def display_homunculus_parts():
             part_description = homunculus_parts_extended[part].split('(')[1].rstrip(')')
             prompt = f"Learn about the key features and functions of the {part} - {part_description}"
             if st.button(f"Explore {part}", key=part):
-                response = chat_with_model(prompt, part)
+                #response = chat_with_model(prompt, part) # GPT
+                response = StreamLLMChatResponse(prompt) # Llama
 
     with st.expander(f"Brain ({homunculus_parts['Brain']})", expanded=False):
         brain_parts = {
@@ -284,15 +336,18 @@ def display_homunculus_parts():
             # Formatting the prompt in markdown style for enhanced learning
             prompt = f"Create a markdown outline with emojis to explain the {part} and its role in the brain: {description}"
             if st.button(f"Explore {part} 🧠", key=part):
-                response = chat_with_model(prompt, part)
-    
+                #response = chat_with_model(prompt, part)
+                response = StreamLLMChatResponse(prompt) # Llama
+
     # Displaying central body parts
     central_parts = ["Neck", "Chest", "Abdomen", "Pelvis"]
     for part in central_parts:
         with st.expander(f"{part} ({homunculus_parts_extended[part]})", expanded=False):
             prompt = f"Learn about the key features and functions of the {part} - {homunculus_parts_extended[part].split(' ')[-1]}"
             if st.button(f"Explore {part} 🧣", key=part):
-                response = chat_with_model(prompt, part)
+                #response = chat_with_model(prompt, part)
+                response = StreamLLMChatResponse(prompt) # Llama
+
                 
     # Displaying symmetric body parts
     symmetric_parts = ["Shoulder", "Upper Arm", "Elbow", "Forearm", "Wrist", "Hand", "Hip", "Thigh", "Knee", "Shin", "Foot"]
@@ -302,12 +357,15 @@ def display_homunculus_parts():
             with st.expander(f"Left {part} ({homunculus_parts_extended[f'Left {part}']})", expanded=False):
                 prompt = f"Learn about the key features and functions of the Left {part} - {homunculus_parts_extended[f'Left {part}'].split(' ')[-1]}"
                 if st.button(f"Explore Left {part} 💪", key=f"Left {part}"):
-                    response = chat_with_model(prompt, f"Left {part}")
+                    #response = chat_with_model(prompt, f"Left {part}")
+                    response = StreamLLMChatResponse(prompt) # Llama
+
         with col2:
             with st.expander(f"Right {part} ({homunculus_parts_extended[f'Right {part}']})", expanded=False):
                 prompt = f"Learn about the key features and functions of the Right {part} - {homunculus_parts_extended[f'Right {part}'].split(' ')[-1]}"
                 if st.button(f"Explore Right {part} 💪", key=f"Right {part}"):
-                    response = chat_with_model(prompt, f"Right {part}")
+                    #response = chat_with_model(prompt, f"Right {part}")
+                    response = StreamLLMChatResponse(prompt) # Llama
 
 
 # Define function to add paper buttons and links
@@ -325,28 +383,36 @@ def add_paper_buttons_and_links():
             link_button_with_emoji("https://arxiv.org/abs/2310.08560", "MemGPT", "🧠💾 Memory OS")
             outline_memgpt = "Memory Hierarchy, Context Paging, Self-directed Memory Updates, Memory Editing, Memory Retrieval, Preprompt Instructions, Semantic Memory, Episodic Memory, Emotional Contextual Understanding"
             if st.button("Discuss MemGPT Features"):
-                chat_with_model("Discuss the key features of MemGPT: " + outline_memgpt, "MemGPT")
+                prompt = "Discuss the key features of MemGPT: " + outline_memgpt
+                #chat_with_model(prompt, "MemGPT")
+                response = StreamLLMChatResponse(prompt) # Llama
 
     with col2:
         with st.expander("AutoGen 🤖🔗", expanded=False):
             link_button_with_emoji("https://arxiv.org/abs/2308.08155", "AutoGen", "🤖🔗 Multi-Agent LLM")
             outline_autogen = "Cooperative Conversations, Combining Capabilities, Complex Task Solving, Divergent Thinking, Factuality, Highly Capable Agents, Generic Abstraction, Effective Implementation"
             if st.button("Explore AutoGen Multi-Agent LLM"):
-                chat_with_model("Explore the key features of AutoGen: " + outline_autogen, "AutoGen")
+                prompt = "Explore the key features of AutoGen: " + outline_autogen
+                #chat_with_model(prompt, "AutoGen")
+                response = StreamLLMChatResponse(prompt) # Llama
 
     with col3:
         with st.expander("Whisper 🔊🧑‍🚀", expanded=False):
             link_button_with_emoji("https://arxiv.org/abs/2212.04356", "Whisper", "🔊🧑‍🚀 Robust STT")
             outline_whisper = "Scaling, Deep Learning Approaches, Weak Supervision, Zero-shot Transfer Learning, Accuracy & Robustness, Pre-training Techniques, Broad Range of Environments, Combining Multiple Datasets"
             if st.button("Learn About Whisper STT"):
-                chat_with_model("Learn about the key features of Whisper: " + outline_whisper, "Whisper")
+                prompt = "Learn about the key features of Whisper: " + outline_whisper
+                #chat_with_model(prompt, "Whisper")
+                response = StreamLLMChatResponse(prompt) # Llama
 
     with col4:
         with st.expander("ChatDev 💬💻", expanded=False):
             link_button_with_emoji("https://arxiv.org/pdf/2307.07924.pdf", "ChatDev", "💬💻 Comm. Agents")
             outline_chatdev = "Effective Communication, Comprehensive Software Solutions, Diverse Social Identities, Tailored Codes, Environment Dependencies, User Manuals"
             if st.button("Deep Dive into ChatDev"):
-                chat_with_model("Deep dive into the features of ChatDev: " + outline_chatdev, "ChatDev")
+                prompt = "Deep dive into the features of ChatDev: " + outline_chatdev
+                #chat_with_model(prompt, "ChatDev")
+                response = StreamLLMChatResponse(prompt) # Llama
 
 add_paper_buttons_and_links()
 
@@ -381,7 +447,8 @@ def create_expanders_and_buttons(content):
             with st.expander(header, expanded=False):
                 if st.button(f"Explore {header}"):
                     expanded_outline = "Expand on the feature: " + detail
-                    chat_with_model(expanded_outline, header)
+                    #chat_with_model(expanded_outline, header)
+                    response = StreamLLMChatResponse(expanded_outline) # Llama
 
 def extract_feature_and_detail(paragraph):
     # Use regex to find the header and detail in the paragraph
@@ -404,7 +471,10 @@ def transcribe_audio(file_path, model):
         response = requests.post(OPENAI_API_URL, headers=headers, files=data, data={'model': model})
     if response.status_code == 200:
         st.write(response.json())
-        chatResponse = chat_with_model(response.json().get('text'), '') # *************************************
+        prompt = response.json().get('text')
+        chatResponse = chat_with_model(prompt, '') # *************************************
+        response = StreamLLMChatResponse(prompt) # Llama
+
         transcript = response.json().get('text')
         #st.write('Responses:')
         #st.write(chatResponse)
@@ -634,7 +704,9 @@ def main():
             else:
                 if st.button(f"Chat about Section {i+1}"):
                     st.write('Reasoning with your inputs...')
-                    response = chat_with_model(user_prompt, section, model_choice)
+                    #response = chat_with_model(user_prompt, section, model_choice)
+                    response = StreamLLMChatResponse(user_prompt + ' ' + section) # Llama
+
                     st.write('Response:')
                     st.write(response)
                     document_responses[i] = response
@@ -650,7 +722,8 @@ def main():
         full_response = ''
         for prompt_section in user_prompt_sections:
             # Process each section with the model
-            response = chat_with_model(prompt_section, ''.join(list(document_sections)), model_choice)
+            #response = chat_with_model(prompt_section, ''.join(list(document_sections)), model_choice)
+            response = StreamLLMChatResponse(prompt_section + ''.join(list(document_sections))) # Llama
             full_response += response + '\n'  # Combine the responses
         response = full_response
         st.write('Response:')
@@ -711,7 +784,9 @@ def main():
         if next_action=='search':
             file_content_area = st.text_area("File Contents:", file_contents, height=500)
             st.write('Reasoning with your inputs...')
-            response = chat_with_model(user_prompt, file_contents, model_choice)
+            #response = chat_with_model(user_prompt, file_contents, model_choice)
+            response = StreamLLMChatResponse(user_prompt + ' ' + file_contents) # Llama
+
             filename = generate_filename(file_contents, choice)
             create_file(filename, user_prompt, response, should_save)