a
    ^g                     @  s   d dl mZ d dlZd dlZd dlZdddddZdddd	d
dZdddddddZdddddddddZddddddZ	dddddddZ
dS )    )annotationsNstrbytes)blobpathreturnc              
   C  s   |  ds|  dszdd l}W n. tyN } ztd|W Y d }~n
d }~0 0 || d}| W  d    S 1 sz0    Y  dd l}|| }|  |jS )Nzhttp://zhttps://r   Oblobfile is not installed. Please install it by running `pip install blobfile`.rb)	
startswithblobfileImportErrorBlobFilereadrequestsgetraise_for_statuscontent)r   r
   efr   resp r   [/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/tiktoken/load.py	read_file   s    &
r   bool)dataexpected_hashr   c                 C  s   t |  }||kS N)hashlibsha256	hexdigest)r   r   Zactual_hashr   r   r   
check_hash   s    r   z
str | None)r   r   r   c                 C  s  d}dt jv rt jd }n4dt jv r0t jd }ndd l}t j| d}d}|dkr^t| S t| 	 
 }t j||}t j|rt|d}| }W d    n1 s0    Y  |d u st||r|S zt | W n ty   Y n0 t| }	|r$t|	|s$td	|  d
| ddd l}
zlt j|dd |d t|
  d }t|d}||	 W d    n1 s0    Y  t || W n ty   |r Y n0 |	S )NTZTIKTOKEN_CACHE_DIRZDATA_GYM_CACHE_DIRr   zdata-gym-cacheF r   z'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)exist_ok.z.tmpwb)osenvirontempfilepathjoin
gettempdirr   r   sha1encoder   existsopenr   r   removeOSError
ValueErroruuidmakedirsr   uuid4writerename)r   r   Zuser_specified_cache	cache_dirr&   	cache_key
cache_pathr   r   contentsr1   Ztmp_filenamer   r   r   read_file_cached    sJ    

&*r:   zdict[bytes, int])vocab_bpe_fileencoder_json_filevocab_bpe_hashencoder_json_hashr   c                   sF  dd t dD }dd |D  d}t dD ].}||vr,|| | td| < |d7 }q,t|dkslJ t| | }dd |d	dd
 D }ddd fdddd t|D }	t|	}|D ]$\}
}||	|
| < |d7 }qdd l}|	t||}fdd|
 D }|dd  |dd  |	|ksBJ |	S )Nc                 S  s(   g | ] }t | rt |d kr|qS ) )chrisprintable.0br   r   r   
<listcomp>]       z3data_gym_to_mergeable_bpe_ranks.<locals>.<listcomp>   c                 S  s   i | ]}t ||qS r   )r@   rB   r   r   r   
<dictcomp>_   rF   z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>r      c                 S  s   g | ]}t | qS r   )tuplesplit)rC   Z	merge_strr   r   r   rE   j   rF   
r   r   )valuer   c                   s   t  fdd| D S )Nc                 3  s   | ]} | V  qd S r   r   rB   data_gym_byte_to_byter   r   	<genexpr>m   rF   zKdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>r   )rN   rO   r   r   decode_data_gyml   s    z8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gymc                 S  s   i | ]\}}t |g|qS r   rR   )rC   irD   r   r   r   rH   p   rF   c                   s   i | ]\}} ||qS r   r   )rC   kv)rS   r   r   rH   }   rF   s   <|endoftext|>s   <|startoftext|>)rangeappendr@   lenr:   decoderK   	enumeratejsonloadsitemspop)r;   r<   r=   r>   Zrank_to_intbytenrD   Zvocab_bpe_contentsZ
bpe_merges	bpe_ranksfirstsecondr\   Zencoder_jsonZencoder_json_loadedr   )rP   rS   r   data_gym_to_mergeable_bpe_ranksV   s0    


rd   None)ra   tiktoken_bpe_filer   c              
   C  s   zdd l }W n. ty: } ztd|W Y d }~n
d }~0 0 ||dR}t|  dd dD ],\}}|t|d t|	  d  q^W d    n1 s0    Y  d S )	Nr   r   r#   c                 S  s   | d S )NrI   r   )xr   r   r   <lambda>   rF   z#dump_tiktoken_bpe.<locals>.<lambda>)key       
)
r
   r   r   sortedr^   r4   base64	b64encoder   r+   )ra   rf   r
   r   r   tokenrankr   r   r   dump_tiktoken_bpe   s    rq   )rf   r   r   c                 C  s   t | |}i }| D ]h}|s qz"| \}}t||t|< W q ty| } z"td|d|  |W Y d }~qd }~0 0 q|S )NzError parsing line z in )r:   
splitlinesrK   intrm   	b64decode	Exceptionr0   )rf   r   r9   retlinero   rp   r   r   r   r   load_tiktoken_bpe   s    
.rx   )N)NN)N)
__future__r   rm   r   r$   r   r   r:   rd   rq   rx   r   r   r   r   <module>   s   9  0