a bgs@sZddlmZddlZddlZddlZddlmZmZmZm Z m Z m Z m Z m Z mZmZmZmZddlZddlmZddlmZddlmZmZmZddlmZmZmZm Z ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)e*e+Z,d d d d dZ-d dd ddZ.d"ddddddZ/d dddddZ0d dddddZ1edddd Gd!d d eeZ2dS)#) annotationsN) AnyCallableDictListLiteralMappingOptionalSequenceSetTupleUnioncast) deprecated) Embeddings)get_from_dict_or_envget_pydantic_field_namespre_init) BaseModel ConfigDictFieldmodel_validator) AsyncRetryingbefore_sleep_logretryretry_if_exception_typestop_after_attemptwait_exponential) is_openai_v1OpenAIEmbeddingszCallable[[Any], Any]) embeddingsreturncCsnddl}tdt|jtd|j|jdt|jj t|jj Bt|jj Bt|jj Bt|jj BtttjdS)NrT multiplierminmaxreraisestopwaitr before_sleep)openairr max_retriesrretry_min_secondsretry_max_secondsrerrorTimeoutAPIErrorAPIConnectionErrorRateLimitErrorServiceUnavailableErrorrloggerloggingWARNING)r r,r9s/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langchain_community/embeddings/openai.py_create_retry_decorator,s(      r;rcsddl}tdt|jtd|j|jdt|jj t|jj Bt|jj Bt|jj Bt|jj Btttjddddfdd }|S) NrTr"r#r'r)funcr!csddddfdd }|S)Nrr)argskwargsr!cs62z"3dHW}|i|IdHS6tddS)Nzthis is unreachable)AssertionError)r=r>_)async_retryingr<r9r: wrapped_fbsz7_async_retry_decorator..wrap..wrapped_fr9)r<rBrA)r<r:wrapasz$_async_retry_decorator..wrap)r,rrr-rr.r/rr0r1r2r3r4r5rr6r7r8)r r,rDr9rCr:_async_retry_decoratorGs,      rEFdictbool)response skip_emptyr!cCs2tdd|dDr.|s.ddl}|jd|S)Ncss|]}t|ddkVqdS) embeddingr"N)len).0dr9r9r: nz"_check_response..datarz&OpenAI API returned an empty embedding)anyr,r0r2)rHrIr,r9r9r:_check_responsems rR)r r>r!c sFtrjjfi|St}|dddfdd }|fi|S))Use tenacity to retry the embedding call.rr>r!cs jjfi|}t|jdSN)rI)clientcreaterRrIr>rHr r9r:_embed_with_retry{sz+embed_with_retry.._embed_with_retry)rrVrWr;)r r>Zretry_decoratorrZr9rYr:embed_with_retryus r[csNtrjjfi|IdHStdddfdd }|fi|IdHS)rSNrrTcs&jjfi|IdH}t|jdSrU)rVZacreaterRrIrXrYr9r:_async_embed_with_retrysz7async_embed_with_retry.._async_embed_with_retry)r async_clientrWrE)r r>r\r9rYr:async_embed_with_retrys r^z0.0.9z1.0z!langchain_openai.OpenAIEmbeddings)ZsinceZremovalZalternative_importc@sdeZdZUdZedddZded<edddZded<dZd ed <eZ d ed <edd dZ d ed<edddZ d ed<dZ d ed<dZ d ed<dZded<edddZd ed<edddZd ed<eZded<dZded<d Zded!<d"Zded#<edd$dZd%ed&<dZded'<dZd(ed)<dZd ed*<d+Zd(ed,<eed-Zd.ed/<d+Zd(ed0<dZd1ed2<dZd3ed4<d5Z ded6<d7Z!ded8<dZ"d9ed:<e#dd;dd?e&d.dd@dAdBZ'e(dCdCd@dDdEZ)e*d.dFdGdHZ+ddIdJd dKdLdMdNdOZ,ddIdJd dKdLdMdPdQZ-d^dJdKdLdSdTdUZ.d_dJdKdLdSdVdWZ/d dXdYdZd[Z0d dXdYd\d]Z1dS)`rajOpenAI embedding models. To use, you should have the ``openai`` python package installed, and the environment variable ``OPENAI_API_KEY`` set with your API key or pass it as a named parameter to the constructor. Example: .. code-block:: python from langchain_community.embeddings import OpenAIEmbeddings openai = OpenAIEmbeddings(openai_api_key="my-api-key") In order to use the library with Microsoft Azure endpoints, you need to set the OPENAI_API_TYPE, OPENAI_API_BASE, OPENAI_API_KEY and OPENAI_API_VERSION. The OPENAI_API_TYPE must be set to 'azure' and the others correspond to the properties of your endpoint. In addition, the deployment name must be passed as the model parameter. Example: .. code-block:: python import os os.environ["OPENAI_API_TYPE"] = "azure" os.environ["OPENAI_API_BASE"] = "https://Build extra kwargs from additional params that were passed in.r}zFound z supplied twice.z WARNING! z/ is not default parameter. zJ was transferred to model_kwargs. Please confirm that z is what you intended.z Parameters za should be specified explicitly. Instead they were passed in as part of `model_kwargs` parameter.) rgetlist ValueErrorwarningswarnpop intersectionkeys)clsrZall_required_field_namesr field_nameZinvalid_model_kwargsr9r9r: build_extras*   zOpenAIEmbeddings.build_extrarc Cst|dd|d<|dp td|d<t|dddd|d<t|d d dd|d <|dd vrrd }t|d d|d <nd}t|dd|d|d<|dptdptd|d<z ddl}WntytdYn0trl|dd vrtd|d|d|d|d|d|d|d|dd}| dsH|j fi|j |d<| ds|j fi|j |d<n| ds|j |d<n|S) z?Validate that api key and python package exists in environment.rnZOPENAI_API_KEYrhZOPENAI_API_BASEriZOPENAI_API_TYPE)r_rjZ OPENAI_PROXYZazureZazure_adZazureadz 2023-05-15rtirfZOPENAI_API_VERSIONrpZ OPENAI_ORG_IDZOPENAI_ORGANIZATIONrNTCould not import openai python package. Please install it with `pip install openai`.zfIf you have openai>=1.0.0 installed and are using Azure, please use the `AzureOpenAIEmbeddings` class.rwr-r~rr)rmrorgrvr-r~rrrVr])rosgetenvr%r, ImportErrorrrrrZOpenAIr Z AsyncOpenAIZ Embedding)rrZdefault_api_versionr,Z client_paramsr9r9r:validate_environment s|             z%OpenAIEmbeddings.validate_environment)r!c Cstrd|ji|j}n~|j|j|j|j|j|j|j|j d|j}|jdvrX|j |d<|j rz ddl }Wnt yt dYn0|j |j d|_|S)Nrb)rbrwrxrmroZapi_baseZapi_typerdrenginerr)httphttps)rrbr}rwrxrnrprhrirfrcrjr,rproxy)selfZ openai_argsr,r9r9r:_invocation_paramsms4      z#OpenAIEmbeddings._invocation_params)rtz List[str]z Optional[int]zList[List[float]])textsrrtr!cCsg}g}|jp|j}|p|j}|jszddlm}WntyNtdYn0|j|d} t|D]^\} } | j | dd} t dt | |j D]4} | | | |j }| |}|||| qqdnz ddl}WntytdYn0z||}Wn,ty(td d }||}Yn0t|D]t\} } |jd rT| d d } |j | |j|jd}t dt ||j D](} ||| | |j || qzq2|jrz$ddlm}|t dt ||}Wn$tyt dt ||}Yn0nt dt ||}g}|D]T} t|fd|| | |i|j}t|tsN|}| dd|dDqddt t |D}ddt t |D}t t |D]T} |j!rt || dkrĐq||| || ||| t || qddt t |D}t t |D]} || }t |dkrpt|fddi|j}t|ts^|}|ddd}nt"j#|d|| d}|t"j$%|&|| <q|S)al Generate length-safe embeddings for a list of texts. This method handles tokenization and embedding generation, respecting the set embedding context length and chunk size. It supports both tiktoken and HuggingFace tokenizer based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. r AutoTokenizerzCould not import transformers python package. This is needed in order to for OpenAIEmbeddings without `tiktoken`. Please install it with `pip install transformers`. Zpretrained_model_name_or_pathFZadd_special_tokensNCould not import tiktoken python package. This is needed in order to for OpenAIEmbeddings. Please install it with `pip install tiktoken`.5Warning: model not found. Using cl100k_base encoding. cl100k_base001  textrqrs)tqdminputcss|]}|dVqdSrJNr9rLrr9r9r:rNrOz.rPcSsg|]}gqSr9r9rLr@r9r9r: rOz=OpenAIEmbeddings._get_len_safe_embeddings..cSsg|]}gqSr9r9rr9r9r:rrOr"cSsg|]}gqSr9r9rr9r9r:rrOrrJZaxisweights)'rzrbrtry transformersrrfrom_pretrained enumerateencoderangerKrldecodeappendtiktokenencoding_for_modelKeyErrorr6warning get_encodingendswithreplacerqrsr{Z tqdm.autorr[r isinstancerFextendrInpaveragelinalgnormtolist)rrrrttokensindices model_name _chunk_sizer tokenizerir tokenizedj token_chunk chunk_textrencodingrbtokenrZ_iterbatched_embeddingsrHresultsnum_tokens_in_batchr _resultaverage_embeddedrr9r9r:_get_len_safe_embeddingss              z)OpenAIEmbeddings._get_len_safe_embeddingscsFg}g}|jp|j}|p|j}|jszddlm}WntyNtdYn0|j|d} t|D]^\} } | j | dd} t dt | |j D]4} | | | |j }| |}|||| qqdnz ddl}WntytdYn0z||}Wn,ty(td d }||}Yn0t|D]t\} } |jd rT| d d } |j | |j|jd}t dt ||j D](} ||| | |j || qzq2g}|p|j}t dt ||D]Z} t|fd|| | |i|jIdH}t|ts|}|dd|dDqddt t |D}ddt t |D}t t |D]6} ||| || ||| t || q\ddt t |D}t t |D]} || }t |dkrt|fddi|jIdH}t|ts|}|ddd}ntj|d|| d}|tj !|"|| <q|S)a Asynchronously generate length-safe embeddings for a list of texts. This method handles tokenization and asynchronous embedding generation, respecting the set embedding context length and chunk size. It supports both `tiktoken` and HuggingFace `tokenizer` based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. rrzCould not import transformers python package. This is needed in order to for OpenAIEmbeddings without `tiktoken`. Please install it with `pip install transformers`.rFrNrrrrrrrrcss|]}|dVqdSrr9rr9r9r:rNurOz=OpenAIEmbeddings._aget_len_safe_embeddings..rPcSsg|]}gqSr9r9rr9r9r:rwrOz>OpenAIEmbeddings._aget_len_safe_embeddings..cSsg|]}gqSr9r9rr9r9r:rxrOcSsg|]}gqSr9r9rr9r9r:r}rOrrJr)#rzrbrtryrrrrrrrrKrlrrrrrr6rrrrrqrsr^rrrFrrrrrr)rrrrtrrrrrrrrrrrrrrrbrrrHrrr rrrr9r9r:_aget_len_safe_embeddingss                z*OpenAIEmbeddings._aget_len_safe_embeddingsr)rrtr!cCstt|j}|j||dS)aMCall out to OpenAI's embedding endpoint for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. Returns: List of embeddings, one for each text. r)rrarcrrrrtrr9r9r:embed_documentss z OpenAIEmbeddings.embed_documentscs tt|j}|j||dIdHS)aSCall out to OpenAI's embedding endpoint async for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. Returns: List of embeddings, one for each text. rN)rrarcrrr9r9r:aembed_documentss z!OpenAIEmbeddings.aembed_documentsz List[float])rr!cCs||gdS)zCall out to OpenAI's embedding endpoint for embedding query text. Args: text: The text to embed. Returns: Embedding for the text. r)r)rrr9r9r: embed_querys zOpenAIEmbeddings.embed_querycs||gIdH}|dS)zCall out to OpenAI's embedding endpoint async for embedding query text. Args: text: The text to embed. Returns: Embedding for the text. Nr)r)rrr r9r9r: aembed_querys zOpenAIEmbeddings.aembed_query)r)r)2__name__ __module__ __qualname____doc__rrV__annotations__r]rbrcrfrhrirjrlrnrpsetrqrsrtr-rwrxryrzr{rFr}rIr~rr.r/rrZ model_configr classmethodrrrpropertyrrrrrrrr9r9r9r:rsf )                 L$z )F)3 __future__rr7rrtypingrrrrrrr r r r r rnumpyrZlangchain_core._api.deprecationrZlangchain_core.embeddingsrZlangchain_core.utilsrrrZpydanticrrrrtenacityrrrrrrZ langchain_community.utils.openair getLoggerrr6r;rErRr[r^rr9r9r9r:s. 8    &