a
    `g{:                     @  s   d Z ddlmZ ddlZddlZddlmZmZmZm	Z	m
Z
mZmZ ddlmZ ddlmZ ddlmZ ddlmZ erdd	lmZ dd
lmZ G dd dZe ZG dd dZG dd dZe ZdgZdS )a  Make approximate assertions as "expectations" on test results.

This module is designed to be used within test cases decorated with the
`@pytest.mark.decorator` decorator
It allows you to log scores about a test case and optionally make assertions that log as
"expectation" feedback to LangSmith.

Example usage:
    .. code-block:: python

        import pytest
        from langsmith import expect


        @pytest.mark.langsmith
        def test_output_semantically_close():
            response = oai_client.chat.completions.create(
                model="gpt-3.5-turbo",
                messages=[
                    {"role": "system", "content": "You are a helpful assistant."},
                    {"role": "user", "content": "Say hello!"},
                ],
            )
            response_txt = response.choices[0].message.content
            # Intended usage
            expect.embedding_distance(
                prediction=response_txt,
                reference="Hello!",
            ).to_be_less_than(0.9)

            # Score the test case
            matcher = expect.edit_distance(
                prediction=response_txt,
                reference="Hello!",
            )
            # Apply an assertion and log 'expectation' feedback to LangSmith
            matcher.to_be_less_than(1)

            # You can also directly make assertions on values directly
            expect.value(response_txt).to_contain("Hello!")
            # Or using a custom check
            expect.value(response_txt).against(lambda x: "Hello" in x)

            # You can even use this for basic metric logging within tests

            expect.score(0.8)
            expect.score(0.7, key="similarity").to_be_greater_than(0.7)
    )annotationsN)TYPE_CHECKINGAnyCallableLiteralOptionalUnionoverloadclient)run_helpers)	run_trees)utils)EditDistanceConfig)EmbeddingConfigc                   @  s,   e Zd ZdZddddZddddZd	S )
_NULL_SENTRYzA sentinel singleton class used to distinguish omitted keyword arguments
    from those passed in with the value None (which may have different behavior).
    zLiteral[False]returnc                 C  s   dS )NF selfr   r   _/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langsmith/_expect.py__bool__P   s    z_NULL_SENTRY.__bool__strc                 C  s   dS )N	NOT_GIVENr   r   r   r   r   __repr__S   s    z_NULL_SENTRY.__repr__N)__name__
__module____qualname____doc__r   r   r   r   r   r   r   K   s   r   c                   @  s   e Zd ZdZd,ddddddd	d
Zd-ddddddZdddddddZdddddZdddddZddddddZ	d.dddddd Z
dddd!d"Zdd#d$d%Zdddd&d'Zd(dd)d*d+ZdS )/_Matcherz4A class for making assertions on expectation values.NOptional[ls_client.Client]r   r   z,Optional[ls_utils.ContextThreadPoolExecutor]Optional[str])r   keyvalue	_executorrun_idc                 C  s@   || _ || _|| _|p tjdd| _t }|r6|jn|| _	d S )N   max_workers)
_clientr#   r$   ls_utilsContextThreadPoolExecutorr%   rhget_current_run_treetrace_id_run_id)r   r   r#   r$   r%   r&   rtr   r   r   __init__]   s    z_Matcher.__init__intNone)scoremessager   c                 C  s8   t  s4| jst | _| jj| jj| jd||d d S )NZexpectation)r&   r#   r5   comment)	r+   test_tracking_is_disabledr*   r1   get_cached_clientr%   submitcreate_feedbackr0   )r   r5   r6   r   r   r   _submit_feedbackl   s    
z_Matcher._submit_feedbackbool)	conditionr6   method_namer   c              
   C  sl   z,|sJ || j dd| j d| d W n: tyf } z"|  dt| |d W Y d }~n
d }~0 0 d S )N   z	Success: .)r6   r   )r<   r#   AssertionErrorrepr)r   r>   r6   r?   er   r   r   _assertx   s     z_Matcher._assertfloatr$   r   c              	   C  s.   |  | j|k d| j d| d| j d dS )zAssert that the expectation value is less than the given value.

        Args:
            value: The value to compare against.

        Raises:
            AssertionError: If the expectation value is not less than the given value.
        	Expected z to be less than 
, but got to_be_less_thanNrE   r$   r#   r   r$   r   r   r   rJ      s
    	z_Matcher.to_be_less_thanc              	   C  s.   |  | j|kd| j d| d| j d dS )a  Assert that the expectation value is greater than the given value.

        Args:
            value: The value to compare against.

        Raises:
            AssertionError: If the expectation value is not
            greater than the given value.
        rH   z to be greater than rI   to_be_greater_thanNrK   rL   r   r   r   rM      s
    
z_Matcher.to_be_greater_than)	min_value	max_valuer   c                 C  sD   |  || j  k o|k n  d| j d| d| d| j d dS )aJ  Assert that the expectation value is between the given min and max values.

        Args:
            min_value: The minimum value (exclusive).
            max_value: The maximum value (exclusive).

        Raises:
            AssertionError: If the expectation value
                is not between the given min and max.
        rH   z to be between z and rI   to_be_betweenNrK   )r   rN   rO   r   r   r   rP      s    z_Matcher.to_be_between   )r$   	precisionr   c              	   C  s:   |  t| j|t||kd| j d| d| j d dS )ak  Assert that the expectation value is approximately equal to the given value.

        Args:
            value: The value to compare against.
            precision: The number of decimal places to round to for comparison.

        Raises:
            AssertionError: If the rounded expectation value
                does not equal the rounded given value.
        rH   z to be approximately rI   to_be_approximatelyN)rE   roundr$   r#   )r   r$   rR   r   r   r   rS      s
    z_Matcher.to_be_approximatelyc              	   C  s.   |  | j|kd| j d| d| j d dS )a   Assert that the expectation value equals the given value.

        Args:
            value: The value to compare against.

        Raises:
            AssertionError: If the expectation value does
                not exactly equal the given value.
        rH   z to be equal to rI   to_equalNrK   rL   r   r   r   rU      s
    
z_Matcher.to_equalr   c                 C  s(   |  | jdu d| j d| j d dS )zAssert that the expectation value is None.

        Raises:
            AssertionError: If the expectation value is not None.
        NrH   z to be None, but got 
to_be_nonerK   r   r   r   r   rV      s
    z_Matcher.to_be_nonec                 C  s(   |  || jv d| j d| dd dS )zAssert that the expectation value contains the given value.

        Args:
            value: The value to check for containment.

        Raises:
            AssertionError: If the expectation value does not contain the given value.
        rH   z to contain z, but it does not
to_containNrK   rL   r   r   r   rW      s
    	z_Matcher.to_containr   )funcr   c                C  s0   t |}| || jd| d| j d dS )zAssert the expectation value against a custom function.

        Args:
            func: A custom function that takes the expectation value as input.

        Raises:
            AssertionError: If the custom function returns False.
        z
Assertion z failed for againstN)inspect	signaturerE   r$   r#   )r   rX   Zfunc_signaturer   r   r   rY      s    	
z_Matcher.against)NN)N)rQ   )r   r   r   r   r2   r<   rE   rJ   rM   rP   rS   rU   rV   rW   rY   r   r   r   r   r    Z   s     r    c                   @  s   e Zd ZdZddddddZddddd	d
dddZdddddd
dddZdd
dddZddddddddd
dddZe	dd
dddZ
e	dd d d!dZ
edfdd"d#d$d%dZ
dd&d'd(d)ZdS )*_Expectz1A class for setting expectations on test results.Nr
   r!   c                C  s*   || _ tjdd| _tj| jjdd d S )Nr'   r(   T)wait)r*   r+   r,   executoratexitregistershutdownr   r   r   r   r   r2     s    z_Expect.__init__configr   zOptional[EmbeddingConfig]r    )
prediction	referencerd   r   c          	   	   C  s   ddl m} |pi }|dr"dnd}||d}|j||d}||jd}| d	||d
| d|j d t| jd	|| jdS )a  Compute the embedding distance between the prediction and reference.

        This logs the embedding distance to LangSmith and returns a `_Matcher` instance
        for making assertions on the distance value.

        By default, this uses the OpenAI API for computing embeddings.

        Args:
            prediction: The predicted string to compare.
            reference: The reference string to compare against.
            config: Optional configuration for the embedding distance evaluator.
                Supported options:
                - `encoder`: A custom encoder function to encode the list of input
                     strings to embeddings. Defaults to the OpenAI API.
                - `metric`: The distance metric to use for comparison.
                    Supported values: "cosine", "euclidean", "manhattan",
                    "chebyshev", "hamming".

        Returns:
            A `_Matcher` instance for the embedding distance value.


        Examples:
            >>> expect.embedding_distance(
            ...     prediction="hello",
            ...     reference="hi",
            ... ).to_be_less_than(1.0)
        r   )EmbeddingDistanceencoderZcustomZopenairc   re   rf   )rh   metricembedding_distanceUsing z
, Metric: r5   source_infor7   r%   )	'langsmith._internal._embedding_distancerg   getevaluateZdistancer<   r    r*   r^   )	r   re   rf   rd   rg   Zencoder_func	evaluatorr5   src_infor   r   r   rk     s     #
z_Expect.embedding_distancezOptional[EditDistanceConfig]c          
   	   C  s   ddl m} |pi }|dp d}|dd}||d}|j||d}||d	}	| d
||	d| d| d t| jd
|| jdS )a  Compute the string distance between the prediction and reference.

        This logs the string distance (Damerau-Levenshtein) to LangSmith and returns
        a `_Matcher` instance for making assertions on the distance value.

        This depends on the `rapidfuzz` package for string distance computation.

        Args:
            prediction: The predicted string to compare.
            reference: The reference string to compare against.
            config: Optional configuration for the string distance evaluator.
                Supported options:
                - `metric`: The distance metric to use for comparison.
                    Supported values: "damerau_levenshtein", "levenshtein",
                    "jaro", "jaro_winkler", "hamming", "indel".
                - `normalize_score`: Whether to normalize the score between 0 and 1.

        Returns:
            A `_Matcher` instance for the string distance value.

        Examples:
            >>> expect.edit_distance("hello", "helo").to_be_less_than(1)
        r   )EditDistancerj   Zdamerau_levenshteinZnormalize_scoreTrc   ri   )rj   	normalizeedit_distancerl   z, Normalize: rm   ro   )"langsmith._internal._edit_distanceru   rq   rr   r<   r    r*   r^   )
r   re   rf   rd   ru   rj   rv   rs   r5   rt   r   r   r   rw   =  s(    

z_Expect.edit_distancer   rG   c                 C  s   t | jd|| jdS )a"  Create a `_Matcher` instance for making assertions on the given value.

        Args:
            value: The value to make assertions on.

        Returns:
            A `_Matcher` instance for the given value.

        Examples:
           >>> expect.value(10).to_be_less_than(20)
        r$   ro   )r    r*   r^   rL   r   r   r   r$   r  s    z_Expect.valuer5   )r#   source_run_idr7   zUnion[float, int, bool]zOptional[ls_client.ID_TYPE]r"   )r5   r#   ry   r7   r   c                C  s.   |  ||ddi||d t| j||| jdS )a  Log a numeric score to LangSmith.

        Args:
            score: The score value to log.
            key: The key to use for logging the score. Defaults to "score".

        Examples:
            >>> expect.score(0.8)  # doctest: +ELLIPSIS
            <langsmith._expect._Matcher object at ...>

            >>> expect.score(0.8, key="similarity").to_be_greater_than(0.7)
        methodzexpect.score)r5   rn   ry   r7   ro   )r<   r    r*   r^   )r   r5   r#   ry   r7   r   r   r   r5     s    	z_Expect.scorec                C  s   d S Nr   rL   r   r   r   __call__  s    z_Expect.__call__zls_client.Client)r   r   c               C  s   d S r{   r   rb   r   r   r   r|     s    zOptional[Any]zUnion[_Expect, _Matcher])r   r$   r   c                C  s    t |d}|tur||S |S )Nr
   )r\   r   r$   )r   r$   r   expectedr   r   r   r|     s    

dict)r#   resultsc                 C  sP   t  }|r|jnd }t sL| js.t | _| jj	| jj
f||d| d S )N)r&   r#   )r-   r.   r/   r+   r8   r*   r1   r9   r^   r:   r;   )r   r#   r   Zcurrent_runr&   r   r   r   r<     s    
z_Expect._submit_feedback)r   r   r   r   r2   rk   rw   r$   r5   r	   r|   r   r<   r   r   r   r   r\      s&   
;5!r\   expect) r   
__future__r   r_   rZ   typingr   r   r   r   r   r   r	   Z	langsmithr   Z	ls_clientr   r-   r   r1   r   r+   rx   r   rp   r   r   r   r    r\   r   __all__r   r   r   r   <module>   s&   1$
 & ?