Update app.py

app.py

@@ -4,33 +4,79 @@ import requests
 
 import gradio as gr
 
+STYLE = """
+.group-border {
+  padding: 10px;
+  border-width: 1px;
+  border-radius: 10px;
+  border-color: gray;
+  border-style: dashed;
+  box-shadow: 1px 1px 3px;
+}
+.control-label-font {
+  font-size: 13pt !important;
+}
+.control-button {
+  background: none !important;
+  border-color: #69ade2 !important;
+  border-width: 2px !important;
+  color: #69ade2 !important;
+}
+.center {
+  text-align: center;
+}
+.right {
+  text-align: right;
+}
+.no-label {
+  padding: 0px !important;
+}
+.no-label > label > span {
+  display: none;
+}
+.small-big {
+  font-size: 12pt !important;
+}
+
+"""
+
 def avaliable_providers():
     providers = []
-    
+
     headers = {
         "Content-Type": "application/json",
     }
-    endpoint_url = "https://api.endpoints.huggingface.cloud/provider"
+    endpoint_url = "https://api.endpoints.huggingface.cloud/v2/provider"
     response = requests.get(endpoint_url, headers=headers)
 
-    for provider in response.json()['items']:
+    providers = {}
+
+    for provider in response.json()['vendors']:
         if provider['status'] == 'available':
-            providers.append(provider['vendor'])
-    
+            regions = {}
+
+            availability = False
+            for region in provider['regions']:
+                if region["status"] == "available":
+                    regions[region['name']] = {
+                        "label": region['label'],
+                        "computes": region['computes']
+                    }
+                    availability = True
+
+            if availability:
+                providers[provider['name']] = regions
+
     return providers
 
+providers = avaliable_providers()
+
 def update_regions(provider):
     avalialbe_regions = []
-    
-    headers = {
-        "Content-Type": "application/json",
-    }
-    endpoint_url = f"https://api.endpoints.huggingface.cloud/provider/{provider}/region"
-    response = requests.get(endpoint_url, headers=headers)
+    regions = providers[provider]
 
-    for region in response.json()['items']:
-        if region['status'] == 'available':
-            avalialbe_regions.append(f"{region['region']}/{region['label']}")
+    for region, attributes in regions.items():
+        avalialbe_regions.append(f"{region}[{attributes['label']}]")
 
     return gr.Dropdown.update(
         choices=avalialbe_regions,
@@ -38,28 +84,22 @@ def update_regions(provider):
     )
 
 def update_compute_options(provider, region):
-    region = region.split("/")[0]
     avalialbe_compute_options = []
+    computes = providers[provider][region.split("[")[0].strip()]["computes"]
 
-    headers = {
-        "Content-Type": "application/json",
-    }
-    endpoint_url = f"https://api.endpoints.huggingface.cloud/provider/{provider}/region/{region}/compute"
-    print(endpoint_url)
-    response = requests.get(endpoint_url, headers=headers)
-
-    for compute in response.json()['items']:
+    for compute in computes:
         if compute['status'] == 'available':
             accelerator = compute['accelerator']
             numAccelerators = compute['numAccelerators']
             memoryGb = compute['memoryGb']
             architecture = compute['architecture']
             instanceType = compute['instanceType']
-            
+            pricePerHour = compute['pricePerHour']
+
             type = f"{numAccelerators}vCPU {memoryGb} · {architecture}" if accelerator == "cpu" else f"{numAccelerators}x {architecture}"
-            
+
             avalialbe_compute_options.append(
-                f"{compute['accelerator'].upper()} [{compute['instanceSize']}] · {type} · {instanceType}"
+                f"{compute['accelerator'].upper()} [{compute['instanceSize']}] · {type} · {instanceType} · ${pricePerHour}/hour"
             )
 
     return gr.Dropdown.update(
@@ -77,9 +117,9 @@ def submit(
     task_selector,
     framework_selector,
     compute_selector,
-    min_node_selector, 
-    max_node_selector, 
-    security_selector    
+    min_node_selector,
+    max_node_selector,
+    security_selector
 ):
     compute_resources = compute_selector.split("·")
     accelerator = compute_resources[0][:3].strip()
@@ -89,7 +129,7 @@ def submit(
     size = compute_resources[0][size_l_index : size_r_index].strip()
 
     type = compute_resources[-1].strip()
-    
+
     payload = {
       "accountId": hf_account_input.strip(),
       "compute": {
@@ -107,7 +147,7 @@ def submit(
           "huggingface": {}
         },
         "repository": repository_selector.lower(),
-        "revision": head_sha,
+        "revision": "main",
         "task": task_selector.lower()
       },
       "name": endpoint_name_input.strip(),
@@ -117,7 +157,7 @@ def submit(
       },
       "type": security_selector.lower()
     }
-    
+
     print(payload)
 
     payload = json.dumps(payload)
@@ -127,7 +167,7 @@ def submit(
         "Authorization": f"Bearer {hf_token_input.strip()}",
         "Content-Type": "application/json",
     }
-    endpoint_url = f"https://api.endpoints.huggingface.cloud/endpoint"
+    endpoint_url = f"https://api.endpoints.huggingface.cloud/v2/endpoint"
     print(endpoint_url)
 
     response = requests.post(endpoint_url, headers=headers, data=payload)
@@ -143,215 +183,259 @@ def submit(
     else:
         return f"something went wrong {response.status_code} = {response.text}"
 
-with gr.Blocks() as hf_endpoint:
-    providers = avaliable_providers()
-
-    gr.Markdown(
-    """
-    ## Deploy Stable Diffusion on 🤗 Endpoint
-    ---
-    """)
-    
-    gr.Markdown("""
-    #### Your 🤗 Account ID(Name)
-    """)
-    hf_account_input = gr.Textbox(
-        show_label=False,
-    )
-
-    gr.Markdown("""
-    #### Your 🤗 Access Token
-    """)
-    hf_token_input = gr.Textbox(
-        show_label=False,
-        type="password"
-    )
-
-    gr.Markdown("""    
-    #### Decide the Endpoint name
-    """)
-    endpoint_name_input = gr.Textbox(
-        show_label=False
-    )    
-
-    with gr.Row():
-        gr.Markdown("""    
-        #### Cloud Provider
-        """)
-
-        gr.Markdown("""    
-        #### Cloud Region
-        """)       
-    
-    with gr.Row():
-        provider_selector = gr.Dropdown(
-            choices=providers,
-            interactive=True,
-            show_label=False,
-        )
-        
-        region_selector = gr.Dropdown(
-            [],
-            value="",
-            interactive=True,
-            show_label=False,
-        )
-        
-        provider_selector.change(update_regions, inputs=provider_selector, outputs=region_selector)
-
-    with gr.Row():
-        gr.Markdown("""    
-        #### Target Model
-        """)
-
-        gr.Markdown("""    
-        #### Target Model Version(branch)
-        """)       
-    
-    with gr.Row():
-        repository_selector = gr.Textbox(
-            value="chansung/my-kitty",
-            interactive=False,
-            show_label=False,
+with gr.Blocks(css=STYLE) as hf_endpoint:
+    with gr.Tab("🤗 Inference Endpoint"):
+        gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"])
+
+        with gr.Column(elem_classes=["group-border"]):
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("""## Hugging Face account ID (name)""")
+                    hf_account_input = gr.Textbox(show_label=False, elem_classes=["no-label", "small-big"])
+
+                with gr.Column():
+                    gr.Markdown("## Hugging Face access token")
+                    hf_token_input = gr.Textbox(show_label=False, type="password", elem_classes=["no-label", "small-big"])
+
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("""## Target model
+
+Import a model from the Hugging Face hub""")
+                    repository_selector = gr.Textbox(
+                        value="NousResearch/Nous-Hermes-Llama2-70b",
+                        interactive=False,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+                with gr.Column():
+                    gr.Markdown("""## Target model version(branch)
+
+Specify the branch name""")
+                    revision_selector = gr.Textbox(
+                        value=f"main",
+                        interactive=False,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+        with gr.Column(elem_classes=["group-border"]):
+            with gr.Column():
+                gr.Markdown("""## Endpoint name
+
+Input a name for your new endpoint""")
+                endpoint_name_input = gr.Textbox(show_label=False, elem_classes=["no-label", "small-big"])
+
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("""## Cloud Provider
+
+Choose between Amazon Web Services and Microsoft Azure""")
+                    provider_selector = gr.Dropdown(
+                        choices=providers.keys(),
+                        interactive=True,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+                with gr.Column():
+                    gr.Markdown("""## Cloud Region
+
+Choose one of the regions from each cloud provider""")
+                    region_selector = gr.Dropdown(
+                        [],
+                        value="",
+                        interactive=True,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+            with gr.Row(visible=False):
+                with gr.Column():
+                    gr.Markdown("## Task")
+                    task_selector = gr.Textbox(
+                        value="Text Generation",
+                        interactive=False,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+                with gr.Column():
+                    gr.Markdown("## Framework")
+                    framework_selector = gr.Textbox(
+                        value="PyTorch",
+                        interactive=False,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+            with gr.Column():
+                gr.Markdown("""## Select Compute Instance Type
+
+Select a CPU or GPU accelerated compute option for inference""")
+                compute_selector = gr.Dropdown(
+                    [],
+                    value="",
+                    interactive=True,
+                    show_label=False,
+                    elem_classes=["no-label", "small-big"]
+                )
+
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("""## Min Number of Nodes
+
+Automatically scale the number of replicas based on load and compute usage""")
+                    min_node_selector = gr.Number(
+                        value=1,
+                        interactive=True,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+                with gr.Column():
+                    gr.Markdown("""## Max Number of Nodes
+
+Automatically scale the number of replicas based on load and compute usage""")
+                    max_node_selector = gr.Number(
+                        value=1,
+                        interactive=True,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+                with gr.Column():
+                    gr.Markdown("""## Security Level
+
+Choose your endpoint's level of privacy""")
+                    security_selector = gr.Radio(
+                        choices=["Protected", "Public", "Private"],
+                        value="Public",
+                        interactive=True,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+        with gr.Column(elem_classes=["group-border"]):
+            with gr.Column():
+                gr.Markdown("""## Container Type
+
+Text Generation Inference is an optimized container for text generation task""")
+                _ = gr.Textbox("Text Generation Inference", show_label=False, elem_classes=["no-label", "small-big"])
+
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("""## Custom Cuda Kernels
+
+TGI uses custom kernels to speed up inference for some models. You can try disabling them if you encounter issues.""")
+                    _ = gr.Dropdown(
+                        value="Enabled",
+                        choices=["Enabled", "Disabled"],
+                        interactive=True,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+                with gr.Column():
+                    gr.Markdown("""## Quantization
+
+Quantization can reduce the model size and improve latency, with little degradation in model accuracy.""")
+                    _ = gr.Dropdown(
+                        value="None",
+                        choices=["None", "Bitsandbytes", "GPTQ"],
+                        interactive=True,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("""## Max Input Length (per Query)
+
+Increasing this value can impact the amount of RAM required. Some models can only handle a finite range of sequences.""")
+                    _ = gr.Number(
+                        value=1024,
+                        interactive=True,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+                with gr.Column():
+                    gr.Markdown("""## Max Number of Tokens (per Query)
+
+The larger this value, the more memory each request will consume and the less effective batching can be.""")
+                    _ = gr.Number(
+                        value=1512,
+                        interactive=True,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("""## Max Batch Prefill Tokens
+
+Number of prefill tokens used during continuous batching. It can be useful to adjust this number since the prefill operation is memory-intensive and compute-bound.""")
+                    _ = gr.Number(
+                        value=2048,
+                        interactive=True,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+                with gr.Column():
+                    gr.Markdown("""## Max Batch Total Tokens
+
+Number of tokens that can be passed before forcing waiting queries to be put on the batch. A value of 1000 can fit 10 queries of 100 tokens or a single query of 1000 tokens.""")
+                    _ = gr.Number(
+                        value=None,
+                        interactive=True,
+                        show_label=False,
+                        elem_classes=["no-label", "small-big"]
+                    )
+
+        submit_button = gr.Button(
+            value="Submit",
+            elem_classes=["control-label-font", "control-button"]
         )
 
-        revision_selector = gr.Textbox(
-            value=f"main",
+        status_txt = gr.Textbox(
+            value="any status update will be displayed here",
             interactive=False,
-            show_label=False,
-        )        
-
-    with gr.Row():
-        gr.Markdown("""    
-        #### Task
-        """)
-
-        gr.Markdown("""    
-        #### Framework
-        """)      
-    
-    with gr.Row():
-        task_selector = gr.Textbox(
-            value="Custom",
-            interactive=False,
-            show_label=False,
-        )
-
-        framework_selector = gr.Textbox(
-            value="TensorFlow",
-            interactive=False,
-            show_label=False,
+            elem_classes=["no-label"]
         )
 
-    gr.Markdown("""
-    
-    #### Select Compute Instance Type
-    """)    
-    compute_selector = gr.Dropdown(
-        [],
-        value="",
-        interactive=True,
-        show_label=False,
-    )
-    region_selector.change(update_compute_options, inputs=[provider_selector, region_selector], outputs=compute_selector)
-
-    with gr.Row():
-        gr.Markdown("""    
-        #### Min Number of Nodes
-        """)
-
-        gr.Markdown("""    
-        #### Max Number of Nodes
-        """)
-
-        gr.Markdown("""    
-        #### Security Level
-        """)        
-    
-    with gr.Row():
-        min_node_selector = gr.Number(
-            value=1,
-            interactive=True,
-            show_label=False,
-        )
-
-        max_node_selector = gr.Number(
-            value=1,
-            interactive=True,
-            show_label=False,
-        )
-
-        security_selector = gr.Radio(
-            choices=["Protected", "Public", "Private"],
-            value="Public",
-            interactive=True,
-            show_label=False,
-        )
-    
-    submit_button = gr.Button(
-        value="Submit",
-    )
-
-    status_txt = gr.Textbox(
-        value="any status update will be displayed here",
-        interactive=False
-    )
-
-    submit_button.click(
-        submit, 
-        inputs=[
-            hf_account_input,
-            hf_token_input,
-            endpoint_name_input,
-            provider_selector,
-            region_selector,
-            repository_selector,
-            task_selector,
-            framework_selector,
-            compute_selector,
-            min_node_selector, 
-            max_node_selector, 
-            security_selector],
-        outputs=status_txt)
-
-    gr.Markdown("""    
-    #### Pricing Table(CPU) - 2023/1/11
-    """)
-
-    gr.Dataframe(
-        headers=["provider", "size", "$/h", "vCPUs", "Memory", "Architecture"],
-        datatype=["str", "str", "str", "number", "str", "str"],
-        row_count=8,
-        col_count=(6, "fixed"),
-        value=[
-            ["aws", "small", "$0.06", 1, "2GB", "Intel Xeon - Ice Lake"],
-            ["aws", "medium", "$0.12", 2, "4GB", "Intel Xeon - Ice Lake"],
-            ["aws", "large", "$0.24", 4, "8GB", "Intel Xeon - Ice Lake"],
-            ["aws", "xlarge", "$0.48", 8, "16GB", "Intel Xeon - Ice Lake"],
-            ["azure", "small", "$0.06", 1, "2GB", "Intel Xeon"],
-            ["azure", "medium", "$0.12", 2, "4GB", "Intel Xeon"],
-            ["azure", "large", "$0.24", 4, "8GB", "Intel Xeon"],
-            ["azure", "xlarge", "$0.48", 8, "16GB", "Intel Xeon"],
-        ]
-    )
-
-    gr.Markdown("""    
-    #### Pricing Table(GPU) - 2023/1/11
-    """)    
-
-    gr.Dataframe(
-        headers=["provider", "size", "$/h", "GPUs", "Memory", "Architecture"],
-        datatype=["str", "str", "str", "number", "str", "str"],
-        row_count=6,
-        col_count=(6, "fixed"),
-        value=[
-            ["aws", "small", "$0.60", 1, "14GB", "NVIDIA T4"],
-            ["aws", "medium", "$1.30", 1, "24GB", "NVIDIA A10G"],
-            ["aws", "large", "$4.50", 4, "156B", "NVIDIA T4"],
-            ["aws", "xlarge", "$6.50", 1, "80GB", "NVIDIA A100"],
-            ["aws", "xxlarge", "$7.00", 4, "96GB", "NVIDIA A10G"],
-            ["aws", "xxxlarge", "$45.0", 8, "640GB", "NVIDIA A100"],
-        ]
-    )        
-
-hf_endpoint.launch(enable_queue=True)
+        provider_selector.change(update_regions, inputs=provider_selector, outputs=region_selector)
+        region_selector.change(update_compute_options, inputs=[provider_selector, region_selector], outputs=compute_selector)
+
+        submit_button.click(
+            submit,
+            inputs=[
+                hf_account_input,
+                hf_token_input,
+                endpoint_name_input,
+                provider_selector,
+                region_selector,
+                repository_selector,
+                task_selector,
+                framework_selector,
+                compute_selector,
+                min_node_selector,
+                max_node_selector,
+                security_selector],
+            outputs=status_txt)
+
+    with gr.Tab("AWS"):
+        gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"])
+
+    with gr.Tab("GCP"):
+        gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"])
+
+    with gr.Tab("Azure"):
+        gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"])
+
+    with gr.Tab("Lambdalabs"):
+        gr.Markdown("# Deploy LLM on 🤗 Hugging Face Inference Endpoint", elem_classes=["center"])
+
+hf_endpoint.launch(enable_queue=True, debug=True)