From b3767a1d71f1c8bd45b9ea9a285a9d8b1cb9cd94 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 14:25:12 -0500 Subject: [PATCH 01/16] init --- gui/Makefile | 0 gui/lorax-gui/__init__.py | 18 ++++++++++++++++++ gui/requirements.txt | 0 3 files changed, 18 insertions(+) create mode 100644 gui/Makefile create mode 100644 gui/lorax-gui/__init__.py create mode 100644 gui/requirements.txt diff --git a/gui/Makefile b/gui/Makefile new file mode 100644 index 00000000..e69de29b diff --git a/gui/lorax-gui/__init__.py b/gui/lorax-gui/__init__.py new file mode 100644 index 00000000..6d06c75a --- /dev/null +++ b/gui/lorax-gui/__init__.py @@ -0,0 +1,18 @@ +import streamlit as st +import requests + +LORAX_PORT = 8000 +HOST = f"http://localhost:{LORAX_PORT}" + +txt = st.text_area("Enter prompt", "Type Here ...") + +data = { + "inputs": txt, + "parameters": { + "max_new_tokens": 64 + } +} + +if st.button("Generate"): + response = requests.post(f"{HOST}/generate", json=data) + st.write(response.json()["text"]) diff --git a/gui/requirements.txt b/gui/requirements.txt new file mode 100644 index 00000000..e69de29b From 6384443943e57fbf7e8ea1390204bad836857d02 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 15:02:38 -0500 Subject: [PATCH 02/16] remove stuff --- gui/Makefile | 0 gui/lorax-gui/__init__.py | 18 ------------------ gui/requirements.txt | 0 3 files changed, 18 deletions(-) delete mode 100644 gui/Makefile delete mode 100644 gui/lorax-gui/__init__.py delete mode 100644 gui/requirements.txt diff --git a/gui/Makefile b/gui/Makefile deleted file mode 100644 index e69de29b..00000000 diff --git a/gui/lorax-gui/__init__.py b/gui/lorax-gui/__init__.py deleted file mode 100644 index 6d06c75a..00000000 --- a/gui/lorax-gui/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -import streamlit as st -import requests - -LORAX_PORT = 8000 -HOST = f"http://localhost:{LORAX_PORT}" - -txt = st.text_area("Enter prompt", "Type Here ...") - -data = { - "inputs": txt, - "parameters": { - "max_new_tokens": 64 - } -} - -if st.button("Generate"): - response = requests.post(f"{HOST}/generate", json=data) - st.write(response.json()["text"]) diff --git a/gui/requirements.txt b/gui/requirements.txt deleted file mode 100644 index e69de29b..00000000 From 08bf11ee190b27f021838dca76cccc419293bc43 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 15:02:45 -0500 Subject: [PATCH 03/16] fix --- clients/gui/lorax-gui/__init__.py | 36 +++++++++++++++++++++++++++++++ clients/gui/requirements.txt | 3 +++ 2 files changed, 39 insertions(+) create mode 100644 clients/gui/lorax-gui/__init__.py create mode 100644 clients/gui/requirements.txt diff --git a/clients/gui/lorax-gui/__init__.py b/clients/gui/lorax-gui/__init__.py new file mode 100644 index 00000000..c1fee0e7 --- /dev/null +++ b/clients/gui/lorax-gui/__init__.py @@ -0,0 +1,36 @@ +import streamlit as st +import requests +import streamlit_pydantic as sp +from LORAX_TYPES import Parameters + +st.markdown( + r""" + + """, unsafe_allow_html=True +) + +LORAX_PORT = 8080 +HOST = f"http://localhost:{LORAX_PORT}" + +data = sp.pydantic_form(key="my_form", model=Parameters) +txt = st.text_area("Enter prompt", "Type Here ...") + +data = { + "inputs": txt, + "parameters": { + "max_new_tokens": 64 + } +} + +if data: + st.json(data.json()) + +if st.button("Generate"): + response = requests.post(f"{HOST}/generate", json=data) + response.raise_for_status() + resp_data = response.json() + st.write(resp_data['generated_text']) diff --git a/clients/gui/requirements.txt b/clients/gui/requirements.txt new file mode 100644 index 00000000..50f8e813 --- /dev/null +++ b/clients/gui/requirements.txt @@ -0,0 +1,3 @@ +streamlit +lorax-client +streamlit-pydantic \ No newline at end of file From 9aa1d071c3e2b523ca0532a70a8b09fcb79277dd Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 17:06:55 -0500 Subject: [PATCH 04/16] add params --- clients/gui/lorax-gui/app.py | 117 +++++++++++++++++++++++++++++++++ clients/python/lorax/client.py | 20 ++++-- 2 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 clients/gui/lorax-gui/app.py diff --git a/clients/gui/lorax-gui/app.py b/clients/gui/lorax-gui/app.py new file mode 100644 index 00000000..f4a1c232 --- /dev/null +++ b/clients/gui/lorax-gui/app.py @@ -0,0 +1,117 @@ +import streamlit as st +import lorax +from lorax.types import Parameters, MERGE_STRATEGIES, ADAPTER_SOURCES, MAJORITY_SIGN_METHODS, MergedAdapters + +LORAX_PORT = 8080 +HOST = f"http://localhost:{LORAX_PORT}" + +def add_merge_adapter(i): + merge_adapter_id = st.text_input(f"Merge Adapter ID {i+1}", key=f"merge_adapter_id_{i}", value=None, placeholder="Merge Adapter Id") + merge_adapter_weight = st.number_input(f"Merge Adapter Weight {i+1}", key=f"merge_adapter_weight_{i}", value=None, placeholder="Merge Adapter weight") + st.divider() + return merge_adapter_id, merge_adapter_weight + +def render_parameters(params: Parameters): + with st.expander("Request parameters"): + max_new_tokens = st.slider("Max new tokens", 0, 256, 20) + repetition_penalty = st.number_input( + "Repetition penalty", + help="The parameter for repetition penalty. 1.0 means no penalty. See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.", + min_value=0.0, + value=None, + placeholder="No repition penalty by default", + ) + return_full_text = st.checkbox("Return full text", help="Whether to return the full text or just the generated part") + stop_tokens = st.text_input("Stop tokens", help="A comma seperated list of tokens where generation is stopped if the model encounters any of them") + stop_sequences = stop_tokens.split(",") if stop_tokens else [] + seed = st.number_input("Seed", help="Random seed for generation", min_value=0, value=None, placeholder="Random seed for generation") + temperature = st.number_input("Temperature", help="The value used to module the next token probabilities", min_value=0.0, value=None, placeholder="Temperature for generation") + best_of = st.number_input("Best of", help="The number of independently computed samples to generate and then pick the best from", value=None, placeholder="Best of for generation") + best_of_int = int(best_of) if best_of else None + watermark = st.checkbox("Watermark", help="Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226)") + decoder_input_details = st.checkbox("Decoder input details", help="Whether to return the decoder input details") + + do_sample_val = st.checkbox("Do Sample", help="Whether to use sampling or greedy decoding for text generation") + top_k_int, top_p, typical_p = None, None, None + if do_sample_val: + top_k = st.number_input( + "Top K", + help="The number of highest probability vocabulary tokens to keep for top-k-filtering", + value=None, + placeholder="Top K for generation", + format="%d" + ) + top_k_int = int(top_k) if top_k else None + top_p = st.number_input("Top P", help="The cumulative probability of parameter for top-p-filtering", value=None, placeholder="Top P for generation") + typical_p = st.number_input( + "Typical P", + help="The typical decoding mass. See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information", + value=None, + placeholder="Typical P for generation" + ) + params.max_new_tokens = max_new_tokens + params.repetition_penalty = repetition_penalty + params.return_full_text = return_full_text + params.stop_sequences = stop_sequences + params.seed = seed + params.temperature = temperature + params.best_of = best_of_int + params.watermark = watermark + params.decoder_input_details = decoder_input_details + params.do_sample = do_sample_val + params.top_k = top_k_int + params.top_p = top_p + params.typical_p = typical_p + st.write(params) + + + with st.expander("Adapter Configuration"): + adapter_id = st.text_input("Adapter ID", value=None, placeholder="Adapter id") + adapter_source = st.selectbox("Adapter Source", options=ADAPTER_SOURCES, index=None) + api_token = st.text_input("API Token", value=None, placeholder="API token") + if st.checkbox("Merged Adapters"): + num_adapters = st.slider("Number of Merge Adapters", value=1, min_value=1, max_value=10) + merge_strategy = st.selectbox("Merge Strategy", options=MERGE_STRATEGIES, index=None) + majority_sign_method = st.selectbox("Majority Sign Method", options=MAJORITY_SIGN_METHODS, index=None) + density = st.number_input("Density", value=0.0, placeholder="Density") + st.divider() + merge_adapters_list = [add_merge_adapter(i) for i in range(num_adapters)] + merge_adapter_ids, merge_adapter_weights = zip(*merge_adapters_list) + merge_adapters = MergedAdapters( + ids=merge_adapter_ids, + weights=merge_adapter_weights, + density=density, + merge_strategy=merge_strategy, + majority_sign_method=majority_sign_method, + ) + params.merged_adapters = merge_adapters + params.adapter_id = adapter_id + params.adapter_source = adapter_source + params.api_token = api_token + st.write(params) + + +def main(): + st.markdown( + r""" + + """, unsafe_allow_html=True + ) + + st.title("Lorax GUI") + params = Parameters() + render_parameters(params) + + txt = st.text_area("Enter prompt", "Type Here ...") + client = lorax.Client(HOST) + if st.button("Generate"): + resp = client.generate(prompt=txt, **params.dict()) + st.write(resp) + + +if __name__ == "__main__": + main() diff --git a/clients/python/lorax/client.py b/clients/python/lorax/client.py index 130b6c44..c6c7599a 100644 --- a/clients/python/lorax/client.py +++ b/clients/python/lorax/client.py @@ -80,6 +80,7 @@ def generate( typical_p: Optional[float] = None, watermark: bool = False, response_format: Optional[Union[Dict[str, Any], ResponseFormat]] = None, + details: bool = True, decoder_input_details: bool = False, ) -> Response: """ @@ -136,6 +137,8 @@ def generate( } } ``` + details (`bool`): + Return the generation details decoder_input_details (`bool`): Return the decoder input token logprobs and ids @@ -149,7 +152,7 @@ def generate( merged_adapters=merged_adapters, api_token=api_token, best_of=best_of, - details=True, + details=details, do_sample=do_sample, max_new_tokens=max_new_tokens, repetition_penalty=repetition_penalty, @@ -190,6 +193,7 @@ def generate_stream( merged_adapters: Optional[MergedAdapters] = None, api_token: Optional[str] = None, do_sample: bool = False, + details: bool = True, max_new_tokens: int = 20, repetition_penalty: Optional[float] = None, return_full_text: bool = False, @@ -219,6 +223,8 @@ def generate_stream( API token for accessing private adapters do_sample (`bool`): Activate logits sampling + details (`bool`): + Return the generation details max_new_tokens (`int`): Maximum number of generated tokens repetition_penalty (`float`): @@ -266,7 +272,7 @@ def generate_stream( merged_adapters=merged_adapters, api_token=api_token, best_of=None, - details=True, + details=details, decoder_input_details=False, do_sample=do_sample, max_new_tokens=max_new_tokens, @@ -383,6 +389,7 @@ async def generate( typical_p: Optional[float] = None, watermark: bool = False, response_format: Optional[Union[Dict[str, Any], ResponseFormat]] = None, + details: bool = True, decoder_input_details: bool = False, ) -> Response: """ @@ -439,6 +446,8 @@ async def generate( } } ``` + details (`bool`): + Return the generation details decoder_input_details (`bool`): Return the decoder input token logprobs and ids @@ -452,7 +461,7 @@ async def generate( merged_adapters=merged_adapters, api_token=api_token, best_of=best_of, - details=True, + details=details, decoder_input_details=decoder_input_details, do_sample=do_sample, max_new_tokens=max_new_tokens, @@ -488,6 +497,7 @@ async def generate_stream( merged_adapters: Optional[MergedAdapters] = None, api_token: Optional[str] = None, do_sample: bool = False, + details : bool = True, max_new_tokens: int = 20, repetition_penalty: Optional[float] = None, return_full_text: bool = False, @@ -517,6 +527,8 @@ async def generate_stream( API token for accessing private adapters do_sample (`bool`): Activate logits sampling + details (`bool`): + Return the generation details max_new_tokens (`int`): Maximum number of generated tokens repetition_penalty (`float`): @@ -564,7 +576,7 @@ async def generate_stream( merged_adapters=merged_adapters, api_token=api_token, best_of=None, - details=True, + details=details, decoder_input_details=False, do_sample=do_sample, max_new_tokens=max_new_tokens, From e427d949718f48967e09bfb5bf5a570cea704fec Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 17:08:30 -0500 Subject: [PATCH 05/16] rm --- clients/gui/lorax-gui/__init__.py | 36 ------------------------------- 1 file changed, 36 deletions(-) diff --git a/clients/gui/lorax-gui/__init__.py b/clients/gui/lorax-gui/__init__.py index c1fee0e7..e69de29b 100644 --- a/clients/gui/lorax-gui/__init__.py +++ b/clients/gui/lorax-gui/__init__.py @@ -1,36 +0,0 @@ -import streamlit as st -import requests -import streamlit_pydantic as sp -from LORAX_TYPES import Parameters - -st.markdown( - r""" - - """, unsafe_allow_html=True -) - -LORAX_PORT = 8080 -HOST = f"http://localhost:{LORAX_PORT}" - -data = sp.pydantic_form(key="my_form", model=Parameters) -txt = st.text_area("Enter prompt", "Type Here ...") - -data = { - "inputs": txt, - "parameters": { - "max_new_tokens": 64 - } -} - -if data: - st.json(data.json()) - -if st.button("Generate"): - response = requests.post(f"{HOST}/generate", json=data) - response.raise_for_status() - resp_data = response.json() - st.write(resp_data['generated_text']) From 752043c25f8988849f5d3f7641633f14178266e8 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 17:12:37 -0500 Subject: [PATCH 06/16] fix name --- clients/python/lorax/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/python/lorax/types.py b/clients/python/lorax/types.py index 8ad776c7..4ff22a4d 100644 --- a/clients/python/lorax/types.py +++ b/clients/python/lorax/types.py @@ -86,7 +86,7 @@ class Parameters(BaseModel): # Whether to prepend the prompt to the generated text return_full_text: bool = False # Stop generating tokens if a member of `stop_sequences` is generated - stop: List[str] = [] + stop_sequences: List[str] = [] # Random sampling seed seed: Optional[int] # The value used to module the logits distribution. From c3a5c90586dc258306437a369daf4b00755edc89 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 17:14:06 -0500 Subject: [PATCH 07/16] undo --- clients/python/lorax/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/python/lorax/types.py b/clients/python/lorax/types.py index 4ff22a4d..8ad776c7 100644 --- a/clients/python/lorax/types.py +++ b/clients/python/lorax/types.py @@ -86,7 +86,7 @@ class Parameters(BaseModel): # Whether to prepend the prompt to the generated text return_full_text: bool = False # Stop generating tokens if a member of `stop_sequences` is generated - stop_sequences: List[str] = [] + stop: List[str] = [] # Random sampling seed seed: Optional[int] # The value used to module the logits distribution. From 010276afd10270c18e623409bb0a879f935bd3d2 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 17:15:05 -0500 Subject: [PATCH 08/16] work around naming --- clients/gui/lorax-gui/app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clients/gui/lorax-gui/app.py b/clients/gui/lorax-gui/app.py index f4a1c232..673be7dd 100644 --- a/clients/gui/lorax-gui/app.py +++ b/clients/gui/lorax-gui/app.py @@ -108,6 +108,9 @@ def main(): txt = st.text_area("Enter prompt", "Type Here ...") client = lorax.Client(HOST) + params_dict = params.dict() + params_dict['stop_sequences'] = params_dict['stop'] + params_dict.pop('stop') if st.button("Generate"): resp = client.generate(prompt=txt, **params.dict()) st.write(resp) From abe3030bad37152ad4a0425853f725b92ad77b0f Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 17:15:57 -0500 Subject: [PATCH 09/16] aaaah --- clients/gui/lorax-gui/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/gui/lorax-gui/app.py b/clients/gui/lorax-gui/app.py index 673be7dd..9c81686a 100644 --- a/clients/gui/lorax-gui/app.py +++ b/clients/gui/lorax-gui/app.py @@ -52,7 +52,7 @@ def render_parameters(params: Parameters): params.max_new_tokens = max_new_tokens params.repetition_penalty = repetition_penalty params.return_full_text = return_full_text - params.stop_sequences = stop_sequences + params.stop = stop_sequences params.seed = seed params.temperature = temperature params.best_of = best_of_int From ff77707752bd250f40ef7f551276cdcffd83e0a4 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 17:16:57 -0500 Subject: [PATCH 10/16] :( --- clients/gui/lorax-gui/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/gui/lorax-gui/app.py b/clients/gui/lorax-gui/app.py index 9c81686a..d9fdecbc 100644 --- a/clients/gui/lorax-gui/app.py +++ b/clients/gui/lorax-gui/app.py @@ -112,7 +112,7 @@ def main(): params_dict['stop_sequences'] = params_dict['stop'] params_dict.pop('stop') if st.button("Generate"): - resp = client.generate(prompt=txt, **params.dict()) + resp = client.generate(prompt=txt, **params_dict) st.write(resp) From bca6b3df269f9c97dcfa1a062263b67819c89309 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 17:23:46 -0500 Subject: [PATCH 11/16] revert --- clients/python/lorax/client.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/clients/python/lorax/client.py b/clients/python/lorax/client.py index c6c7599a..130b6c44 100644 --- a/clients/python/lorax/client.py +++ b/clients/python/lorax/client.py @@ -80,7 +80,6 @@ def generate( typical_p: Optional[float] = None, watermark: bool = False, response_format: Optional[Union[Dict[str, Any], ResponseFormat]] = None, - details: bool = True, decoder_input_details: bool = False, ) -> Response: """ @@ -137,8 +136,6 @@ def generate( } } ``` - details (`bool`): - Return the generation details decoder_input_details (`bool`): Return the decoder input token logprobs and ids @@ -152,7 +149,7 @@ def generate( merged_adapters=merged_adapters, api_token=api_token, best_of=best_of, - details=details, + details=True, do_sample=do_sample, max_new_tokens=max_new_tokens, repetition_penalty=repetition_penalty, @@ -193,7 +190,6 @@ def generate_stream( merged_adapters: Optional[MergedAdapters] = None, api_token: Optional[str] = None, do_sample: bool = False, - details: bool = True, max_new_tokens: int = 20, repetition_penalty: Optional[float] = None, return_full_text: bool = False, @@ -223,8 +219,6 @@ def generate_stream( API token for accessing private adapters do_sample (`bool`): Activate logits sampling - details (`bool`): - Return the generation details max_new_tokens (`int`): Maximum number of generated tokens repetition_penalty (`float`): @@ -272,7 +266,7 @@ def generate_stream( merged_adapters=merged_adapters, api_token=api_token, best_of=None, - details=details, + details=True, decoder_input_details=False, do_sample=do_sample, max_new_tokens=max_new_tokens, @@ -389,7 +383,6 @@ async def generate( typical_p: Optional[float] = None, watermark: bool = False, response_format: Optional[Union[Dict[str, Any], ResponseFormat]] = None, - details: bool = True, decoder_input_details: bool = False, ) -> Response: """ @@ -446,8 +439,6 @@ async def generate( } } ``` - details (`bool`): - Return the generation details decoder_input_details (`bool`): Return the decoder input token logprobs and ids @@ -461,7 +452,7 @@ async def generate( merged_adapters=merged_adapters, api_token=api_token, best_of=best_of, - details=details, + details=True, decoder_input_details=decoder_input_details, do_sample=do_sample, max_new_tokens=max_new_tokens, @@ -497,7 +488,6 @@ async def generate_stream( merged_adapters: Optional[MergedAdapters] = None, api_token: Optional[str] = None, do_sample: bool = False, - details : bool = True, max_new_tokens: int = 20, repetition_penalty: Optional[float] = None, return_full_text: bool = False, @@ -527,8 +517,6 @@ async def generate_stream( API token for accessing private adapters do_sample (`bool`): Activate logits sampling - details (`bool`): - Return the generation details max_new_tokens (`int`): Maximum number of generated tokens repetition_penalty (`float`): @@ -576,7 +564,7 @@ async def generate_stream( merged_adapters=merged_adapters, api_token=api_token, best_of=None, - details=details, + details=True, decoder_input_details=False, do_sample=do_sample, max_new_tokens=max_new_tokens, From 741ce8e122c30f62afc793378256eb1421cb287f Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 17:25:16 -0500 Subject: [PATCH 12/16] another1 --- clients/gui/lorax-gui/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/clients/gui/lorax-gui/app.py b/clients/gui/lorax-gui/app.py index d9fdecbc..ac38e20a 100644 --- a/clients/gui/lorax-gui/app.py +++ b/clients/gui/lorax-gui/app.py @@ -111,6 +111,7 @@ def main(): params_dict = params.dict() params_dict['stop_sequences'] = params_dict['stop'] params_dict.pop('stop') + params_dict.pop('details') if st.button("Generate"): resp = client.generate(prompt=txt, **params_dict) st.write(resp) From d820ae0ec2b4734df1ba8ddedd038cd73a73d953 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 17:28:40 -0500 Subject: [PATCH 13/16] better rendering --- clients/gui/lorax-gui/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clients/gui/lorax-gui/app.py b/clients/gui/lorax-gui/app.py index ac38e20a..8bbaf7b4 100644 --- a/clients/gui/lorax-gui/app.py +++ b/clients/gui/lorax-gui/app.py @@ -114,7 +114,9 @@ def main(): params_dict.pop('details') if st.button("Generate"): resp = client.generate(prompt=txt, **params_dict) - st.write(resp) + st.write(resp.generated_text) + with st.expander("Response details"): + st.write(resp.details.dict()) if __name__ == "__main__": From 8cc237e20dca8e7bc6d7ce57518b1472d8e168ad Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Sat, 17 Feb 2024 17:35:14 -0500 Subject: [PATCH 14/16] add rust fn outline --- launcher/src/main.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 0f3c673f..5d37630c 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -357,6 +357,15 @@ struct Args { /// Download model weights only #[clap(long, env)] download_only: bool, + + /// Launch testing GUI + /// This will launch a GUI that will allow you to test the model + #[clap(long, env)] + launch_gui: bool, + + /// Gui port + #[clap(long, env)] + gui_port: Option, } #[derive(Debug)] @@ -958,6 +967,10 @@ fn spawn_shards( Ok(()) } +fn spawn_gui(port: u16) -> Result { + tracing::info!("Starting GUI"); +} + fn spawn_webserver( args: Args, shutdown: Arc, From c2d58d0832a52514d87f6b23b5ff912c946dccc6 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Mon, 19 Feb 2024 11:15:35 -0500 Subject: [PATCH 15/16] add docker lines --- Dockerfile | 5 +++++ clients/gui/Makefile | 3 +++ clients/gui/requirements.txt | 3 +-- launcher/src/main.rs | 26 +++++++++++++++++++++++--- 4 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 clients/gui/Makefile diff --git a/Dockerfile b/Dockerfile index 9ae3dae1..a60bdf5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -230,6 +230,11 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins g++ \ && rm -rf /var/lib/apt/lists/* +# Install gui and client +COPY client client + +RUN cd client/gui && \ + make install # Final image FROM base diff --git a/clients/gui/Makefile b/clients/gui/Makefile new file mode 100644 index 00000000..55b7aed7 --- /dev/null +++ b/clients/gui/Makefile @@ -0,0 +1,3 @@ +install: + pip install pip --upgrade + pip install -r ./requirements.txt diff --git a/clients/gui/requirements.txt b/clients/gui/requirements.txt index 50f8e813..6b0df2d3 100644 --- a/clients/gui/requirements.txt +++ b/clients/gui/requirements.txt @@ -1,3 +1,2 @@ streamlit -lorax-client -streamlit-pydantic \ No newline at end of file +lorax-client \ No newline at end of file diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 5d37630c..9a8e536c 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -967,9 +967,29 @@ fn spawn_shards( Ok(()) } -fn spawn_gui(port: u16) -> Result { - tracing::info!("Starting GUI"); -} +// fn spawn_gui(port: u16) -> Result { +// tracing::info!("Starting GUI"); + +// let mut webserver = match Command::new("lorax-router") +// .stdout(Stdio::piped()) +// .stderr(Stdio::piped()) +// .process_group(0) +// .spawn() +// { +// Ok(p) => p, +// Err(err) => { +// tracing::error!("Failed to start webserver: {}", err); +// if err.kind() == io::ErrorKind::NotFound { +// tracing::error!("lorax-router not found in PATH"); +// tracing::error!("Please install it with `make install-router`") +// } else { +// tracing::error!("{}", err); +// } +// return Err(LauncherError::WebserverCannotStart); +// } +// }; +// Ok(webserver) +// } fn spawn_webserver( args: Args, From d23f62196d36afeeca065b8d8dc92e9ee3df7bb6 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Wed, 21 Feb 2024 12:10:02 -0500 Subject: [PATCH 16/16] fix dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a60bdf5d..756eaea2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -231,9 +231,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && rm -rf /var/lib/apt/lists/* # Install gui and client -COPY client client +COPY clients clients -RUN cd client/gui && \ +RUN cd clients/gui && \ make install # Final image