o sh|@s&ddlZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl m Z ddl mZmZddlmZddlmZddlZddlmZmZddlmZmZmZdd lmZdd lmZm Z dd l!m"Z"m#Z#e$d krxddl%Z%e"rdd l&m'Z'ddl(m)Z)ddl*m+Z+e#rddl,Z,ddlm-Z-mZm.Z.mZe/ej0ej1Z2e/ej0ej3ej1dZ4ddiddiddiddiddiddiddidZ5dZ6dd7e58dZ9Gdd d Z:eGd!d"d"Z;d#e fd$d%Zd(kre;Z?d)e?_@d*e?_@e=e?ZAeABdSdS)+N)ArgumentParser Namespace) AsyncIterator) dataclassfield)Thread)Optional)AsyncInferenceClientChatCompletionStreamOutput) AutoTokenizerGenerationConfigPreTrainedTokenizer)BaseTransformersCLICommand)ServeArguments ServeCommand)is_rich_availableis_torch_availableWindows)Console)Live)Markdown)AutoModelForCausalLMr BitsAndBytesConfigr z .!\"#$%&'()*+,\-/:<=>?@[]^_`{|}~textz5There is a Llama in my lawn, how can I get rid of it?zyWrite a Python function that integrates any Python function f(x) numerically over an arbitrary interval [x_start, x_end].z4How many helicopters can a human eat in one sitting?z4Count to 10 but skip every number ending with an 'e'zWhy aren't birds real?z2Why is it important to eat socks after meditating?z$Which number is larger, 9.9 or 9.11?)llamacode helicopternumbersbirdssocksnumbers2a **TRANSFORMERS CHAT INTERFACE** Chat interface to try out a model. Besides chatting with the model, here are some basic commands: - **!help**: shows all available commands (set generation settings, save chat, etc.) - **!status**: shows the current status of the model and generation settings - **!clear**: clears the current conversation and starts a new one - **!exit**: closes the interface am **TRANSFORMERS CHAT INTERFACE HELP** Full command list: - **!help**: shows this help message - **!clear**: clears the current conversation and starts a new one - **!status**: shows the current status of the model and generation settings - **!example {NAME}**: loads example named `{NAME}` from the config and uses it as the user input. Available example names: `z`, `a%` - **!set {ARG_1}={VALUE_1} {ARG_2}={VALUE_2}** ...: changes the system prompt or generation settings (multiple settings are separated by a space). Accepts the same flags and format as the `generate_flags` CLI argument. If you're a new user, check this basic flag guide: https://huggingface.co/docs/transformers/llm_tutorial#common-options - **!save {SAVE_NAME} (optional)**: saves the current chat and settings to file by default to `./chat_history/{MODEL_NAME}/chat_{DATETIME}.yaml` or `{SAVE_NAME}` if provided - **!exit**: closes the interface c@seZdZddeedeefddZdeedeee ffdd Z defd d Z d d Z defddZ dedefddZddefddZdededefddZdS) RichInterfaceN model_name user_namecCs:t|_|dur d|_n||_|durd|_dS||_dS)N assistantuser)r_consoler"r#)selfr"r#r(X/var/www/html/alpaca_bot/venv/lib/python3.10/site-packages/transformers/commands/chat.py__init__ss  zRichInterface.__init__streamreturnc s|jd|jdt|jdd\}d}|IdH2zK3dHW}|jdjj}|s+qtdd|}||7}g}| D]}| || d rN| d q<| d q:)consolerefresh_per_secondrz <(/*)(\w*)>z\<\1\2\>z``` z z github-dark) code_themeT)refresh)r&printr"rchoicesdeltacontentresub splitlinesappend startswithrjoinstripupdate) r'r+livertokenoutputslineslinemarkdownr(r(r) stream_output~s,      (zRichInterface.stream_outputcCs$|jd|jd}|j|S)z!Gets user input from the console. [bold red]: )r&inputr#r4)r'rHr(r(r)rHs zRichInterface.inputcCs|jdS)zClears the console.N)r&clearr'r(r(r)rIszRichInterface.clearrcCs(|jd|jd||jdS)z%Prints a user message to the console.rGz>:[/ bold red] N)r&r4r#)r'rr(r(r)print_user_messagesz RichInterface.print_user_messagecolorcCs&|jd|d||jdS)z,Prints text in a given color to the console.z[bold ]Nr&r4)r'rrLr(r(r) print_colorzRichInterface.print_colorFminimalcCs&|jt|rtnt|jdS)z'Prints the help message to the console.N)r&r4rHELP_STRING_MINIMAL HELP_STRING)r'rQr(r(r) print_helprPzRichInterface.print_helpgeneration_config model_kwargscCsJ|jd|d|r|jd||jd||jdS)zFPrints the status of the model and generation settings to the console.z[bold blue]Model: r1z[bold blue]Model kwargs: z [bold blue]NrN)r'r"rUrVr(r(r) print_statuss zRichInterface.print_status)NN)F)__name__ __module__ __qualname__rstrr*rr tupleintrFrHrIrKrOboolrTr dictrWr(r(r(r)r!rs .r!c@seZdZUdZedddidZeeed<edddidZ eeed<eddd idZ eeed <ed dd idZ eed <edddidZ eeed<edddidZ eed<edddidZeeed<edddidZeed<edddidZeed<eddgdddZeeed<eddgdddZeeed <eddd!idZeed"<eddd#idZeeed$<eddd%idZeed&<eddd'idZeed(<ed)d*d+d)gddZeed,<eddd-idZeed.<ed/dd0idZeed1<ed2dd3idZeed4<d5d6ZdS)7 ChatArgumentsz Arguments for the chat CLI. See the metadata arg for each argument's description -- the medatata will be printed with `transformers chat --help` Nhelpz_Name of the pre-trained model. The positional argument will take precedence if both are passed.)defaultmetadatamodel_name_or_pathzKUsername to display in chat interface. Defaults to the current user's name.r%zSystem prompt. system_promptz./chat_history/zFolder to save chat history. save_folderz"Path to a yaml file with examples. examples_pathFz7Whether to show runtime warnings in the chat interface.verbosezPath to a local generation config file or to a HuggingFace repo containing a `generation_config.json` file. Other generation settings passed as CLI arguments will be applied on top of this generation config.rUmainzLSpecific model version to use (can be a branch name, tag name or commit id).model_revisionautozDevice to use for inference.devicezA`torch_dtype` is deprecated! Please use `dtype` argument instead.)rkbfloat16float16float32)rar5 torch_dtypezOverride the default `torch.dtype` and load the model under this dtype. If `'auto'` is passed, the dtype will be automatically derived from the model's weights.dtypez2Whether to trust remote code when loading a model.trust_remote_codezWhich attention implementation to use; you can run --attn_implementation=flash_attention_2, in which case you must install this manually by running `pip install flash-attn --no-build-isolation`.attn_implementationzIWhether to use 8 bit precision for the base model - works only with LoRA. load_in_8bitzIWhether to use 4 bit precision for the base model - works only with LoRA. load_in_4bitnf4zQuantization type.fp4bnb_4bit_quant_typez#Whether to use nested quantization.use_bnb_nested_quant localhostz%Interface the server will listen to..host@zPort the server will listen to.portcCs(|jdur|jdkr|j|_dSdSdS)z(Only used for BC `torch_dtype` argument.Nrk)rprqrJr(r(r) __post_init__!s zChatArguments.__post_init__)rXrYrZ__doc__rrdrr[__annotations__r%rerfrgrhr^rUrjrlrprqrrrsrtrurxryr{r}r]r~r(r(r(r)r`sv   r`argscCst|S)z; Factory function used to chat with a local model. ) ChatCommand)rr(r(r)chat_command_factory(src@sreZdZedefddZddZedefddZed/d e d e edefd d Z ed/de ede e fddZde ede fddZd e dedeee ffddZededede ede edeee eff ddZede de dfdd Zd e ded!effd"d#Zd$ed e d%ed&e ee eeffded'e d(e e dee e ee ffd)d*Zd+d,Zd-d.Zd S)0rparsercCsTtf}|jd|d}|d}|jdtddd|jdtdd d d |jtd dS) z Register this command to argparse so it's available for the transformer-cli Args: parser: Root parser to register command-specific arguments chat)dataclass_typeszPositional argumentsmodel_name_or_path_or_addressNz7Name of the pre-trained model or address to connect to.)typerbragenerate_flagsaFlags to pass to `generate`, using a space as a separator between flags. Accepts booleans, numbers, and lists of integers, more advanced parameterization should be set through --generation-config. Example: `transformers chat max_new_tokens=100 do_sample=False eos_token_id=[1,2]`. If you're a new user, check this basic flag guide: https://huggingface.co/docs/transformers/llm_tutorial#common-options*)rrbranargs)func)r` add_parseradd_argument_group add_argumentr[ set_defaultsr)rr chat_parsergroupr(r(r)register_subcommand0s"  zChatCommand.register_subcommandcCs|jdurD|j}|ds|ds|dr=d|_|jdks$|jdkr(td|jdd\|_|_|jdurs.z4ChatCommand.parse_generate_flags..cSs*i|]\}}||dvr|n|qS))truefalse)lowerrkvr(r(r)rscSs"i|] \}}||dkr dn|qS)Nonenullr(rr(r(r)rs"sr,cSs(|dr |dd}|dddS)N-r.r0)r<replaceisdigit)rr(r(r) is_numbers  z3ChatCommand.parse_generate_flags..is_numbercs*i|]\}}||sd|dn|qS)rr(rrr(r)rs*z, cSsg|] \}}|d|qS)z: r(rr(r(r) sz4ChatCommand.parse_generate_flags..{}z"null"rz"true"rz"false"rz"[[z]"rMrrzFailed to convert `generate_flags` into a valid JSON object. `generate_flags` = {generate_flags} Converted JSON string = {generate_flags_string}) lenitemsr[r^r=rrloadsJSONDecodeErrorr)r'rgenerate_flags_as_dictgenerate_flags_stringprocessed_generate_flagsr(rr)parse_generate_flagss2         z ChatCommand.parse_generate_flagsmodel_generation_configcCs|jdur&d|jvrtj|j}tj|j}t||}nt|j}nt|}|j diddd| |j }|j di|}||fS)zj Returns a GenerationConfig object holding the generation parameters for the CLI command. NrT) do_samplemax_new_tokensr() rUrrrbasenamer from_pretrainedcopydeepcopyr?rr)r'rrrrrUparsed_generate_flagsrVr(r(r)get_generation_parameterizations    z+ChatCommand.get_generation_parameterization tokenizerrU eos_tokens eos_token_idscCs||jdur |j}n|j}g}|dur|||d|dur.|dd|dDt|dkr:||j||fS)z:Retrieves the pad token ID and all possible EOS token IDs.N,cSsg|]}t|qSr()r])rtoken_idr(r(r)rsz0ChatCommand.parse_eos_tokens..r) pad_token_id eos_token_idextendconvert_tokens_to_idsrrr;)rrUrrrall_eos_token_idsr(r(r)parse_eos_tokenss   zChatCommand.parse_eos_tokens model_argsrcCs@|jrtd|j|j|j|jd}|S|jrtdd}|Sd}|S)NT)rubnb_4bit_compute_dtyperxbnb_4bit_use_double_quantbnb_4bit_quant_storage)rt)rurrqrxryrt)rquantization_configr(r(r)get_quantization_config s z#ChatCommand.get_quantization_configrcCstj|j|j|jd}|jdvr|jntt|j}||}|j|j |d|d}t j|jfd|ji|}t|dddurC| |j }||fS)N)revisionrr)rkNrk)rrsrq device_maprrr hf_device_map) r rmodel_name_or_path_positionalrjrrrqgetattrtorchrrsrtorl)r'rrrqrrVmodelr(r(r)load_model_and_tokenizer s.  z$ChatCommand.load_model_and_tokenizer user_input interfaceexamplesrVrcCsd}|dkr||j}|n|dkr|n|drKt|dkrK|} t| dkr6| d} nd} |||| } |jd| d d d n|d r|d d } | } | D]} d| vrq|jd| ddd nq^| | } |j di| }|j di|ng|drt|dkr|d}||vr|g}| ||d| d||ddn4d|dt|d}|j|dd n|dkr|j|j||dnd}|jd|ddd |||||fS)z Handles all user commands except for `!exit`. May update the chat history (e.g. reset it) or the generation config (e.g. set a new flag). Tz!clearz!helpz!saverNzChat saved in !green)rrLz!setr-rz(Invalid flag format, missing `=` after `z;`. Please use the format `arg_1=value_1 arg_2=value_2 ...`.red!examplerr%rzExample z* not found in list of available examples: rz!status)r"rUrVF'z/' is not a valid command. Showing help message.r()rrerIrTr<rrrrOr>rr?rKr;listkeysrWrd)r'rrrrrUrVr valid_command split_inputrnew_generate_flagsrparsed_new_generate_flagsnew_model_kwargs example_name example_errorr(r(r)handle_non_exit_user_commands;s`          z)ChatCommand.handle_non_exit_user_commandscCst|dSN)asynciorun _inner_runrJr(r(r)rszChatCommand.runc sr|jr8t|jj|jj|jj|jj|jj|jj|jj |jj |jj |jj dd }t |}t|jd}d|_||jjd|jj}|jj dkrJdn|jj }t|d|jj }|j}|jdurdt}nt|j } t| }Wdn1sywY|jdur|} n|j} t|j} ||| \} } t|j| d }| |!|j"}|j#dd  z~za|$}|%d r|d krWW|&IdHdS|j'||||| | |d \}}} } |r|%dsWW|&IdHqn|(d|d|j)|d| *|dd}|+|IdH}|(d|dWnt,y&YW|&IdHdSwW|&IdHn|&IdHwq)Nerror) rlrqrrrsrtrurxryr{r} log_level)targetT@rzzhttp://localhostr)r"r#)rQrz!exit)rrrrrUrVrr r%r)rUr)r+ extra_bodyr$)-rrrrlrqrrrsrtrurxryr{r}rrrdaemonstartrdrjr rgDEFAULT_EXAMPLESryaml safe_loadr%rr rrr!rIrrerTrHr<closerr;chat_completionto_json_stringrFKeyboardInterrupt)r' serve_args serve_commandthreadrr{clientrrrr%rrUrVrrrrr+ model_outputr(r(r)rs          !   zChatCommand._inner_runr)rXrYrZ staticmethodrrr*r[rr`rrr r_rrr r\rr r]rrr rr!rrrr(r(r(r)r/sh ' 6    T r__main__z meta-llama/Llama-3.2-3b-Instructzhttp://localhost:8000)Crrrrrr8stringrargparserrcollections.abcr dataclassesrr threadingrtypingrr#huggingface_hubr r transformersr r r transformers.commandsrtransformers.commands.servingrrtransformers.utilsrrrr rich.consoler rich.liver rich.markdownrrrrset ascii_letters whitespaceALLOWED_KEY_CHARSdigitsALLOWED_VALUE_CHARSr"rRr=rrSr!r`rrrXrrrrr(r(r(r)sz         \Y @