
    |Mg                    d    d dl mZ d dlZd dlZd dlZddZddZdddZ	 	 dddZddZ	dd dZ
dS )!    )annotationsNblobpathstrreturnbytesc                   |                      d          s|                      d          sj	 dd l}n"# t          $ r}t          d          |d }~ww xY w|                    | d          5 }|                                cd d d            S # 1 swxY w Y   dd l}|                    |           }|                                 |j        S )Nzhttp://zhttps://r   Oblobfile is not installed. Please install it by running `pip install blobfile`.rb)	
startswithblobfileImportErrorBlobFilereadrequestsgetraise_for_statuscontent)r   r   efr   resps         ?/var/www/api/venv/lib/python3.11/site-packages/tiktoken/load.py	read_filer      s$   y)) (2E2Ej2Q2Q 	OOOO 	 	 	a 	 x.. 	!6688	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 OOO<<!!D<s&   1 
AAA)B

BBdataexpected_hashboolc                Z    t          j        |                                           }||k    S N)hashlibsha256	hexdigest)r   r   actual_hashs      r   
check_hashr"      s(    .&&0022K-''    
str | Nonec                   d}dt           j        v rt           j        d         }nYdt           j        v rt           j        d         }n8dd l}t           j                            |                                d          }d}|dk    rt          |           S t          j        | 	                                          
                                }t           j                            ||          }t           j                            |          rvt          |d          5 }|                                }d d d            n# 1 swxY w Y   |t          ||          r|S 	 t          j        |           n# t           $ r Y nw xY wt          |           }	|r&t          |	|          st#          d	|  d
| d          dd l}
	 t          j        |d           |dz   t)          |
                                          z   dz   }t          |d          5 }|                    |	           d d d            n# 1 swxY w Y   t          j        ||           n# t           $ r |r Y nw xY w|	S )NTTIKTOKEN_CACHE_DIRDATA_GYM_CACHE_DIRr   zdata-gym-cacheF r
   z'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)exist_ok.z.tmpwb)osenvirontempfilepathjoin
gettempdirr   r   sha1encoder    existsopenr   r"   removeOSError
ValueErroruuidmakedirsr   uuid4writerename)r   r   user_specified_cache	cache_dirr.   	cache_key
cache_pathr   r   contentsr9   tmp_filenames               r   read_file_cachedrD       s   rz))J34				+	+J34		GLL!4!4!6!68HII	$B"""X__..//99;;Ii33J	w~~j!! 
*d## 	q6688D	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 Jt]$C$C K	Ij!!!! 	 	 	D	 ""H 
Z-@@ 
Ih I I= I I I
 
 	

 KKK	
I----!C'#djjll*;*;;fD,%% 	GGH	 	 	 	 	 	 	 	 	 	 	 	 	 	 	
	,
++++    		 	
 Os[   D::D>D>E/ /
E<;E<;AI H-!I -H11I 4H15I IIvocab_bpe_fileencoder_json_filevocab_bpe_hashencoder_json_hashdict[bytes, int]c                >   d t          d          D             }d |D             d}t          d          D ]5}||vr/|                    |           |t          d|z             <   |dz  }6t          |          dk    sJ t	          | |                                          }d |                    d          dd         D             }dfdd t          |          D             }	t          |	          }|D ]$\  }
}||	 |
           |          z   <   |dz  }%dd l}|	                    t	          ||                    }fd|
                                D             }|                    dd            |                    dd            |	|k    sJ |	S )Nc                z    g | ]8}t          |                                          #t          |          d k    6|9S ) )chrisprintable.0bs     r   
<listcomp>z3data_gym_to_mergeable_bpe_ranks.<locals>.<listcomp>]   s;    XXXQQ1C1C1E1EX#a&&TW--q---r#      c                .    i | ]}t          |          |S  )rM   rO   s     r   
<dictcomp>z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>_   s     @@@1SVVQ@@@r#   r      c                P    g | ]#}t          |                                          $S rU   )tuplesplit)rP   	merge_strs     r   rR   z3data_gym_to_mergeable_bpe_ranks.<locals>.<listcomp>j   s*    aaay%	))**aaar#   
valuer   r   r   c                :    t          fd| D                       S )Nc              3  (   K   | ]}|         V  d S r   rU   )rP   rQ   data_gym_byte_to_bytes     r   	<genexpr>zKdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>m   s)      ==!*1-======r#   r   )r^   ra   s    r   decode_data_gymz8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gyml   s&    ====u======r#   c                6    i | ]\  }}t          |g          |S rU   rc   )rP   irQ   s      r   rV   z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>p   s&    FFF41asQFFFr#   c                .    i | ]\  }} |          |S rU   rU   )rP   kvrd   s      r   rV   z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>}   s)    RRRTQ??1--qRRRr#   s   <|endoftext|>s   <|startoftext|>)r^   r   r   r   )rangeappendrM   lenrD   decoderZ   	enumeratejsonloadsitemspop)rE   rF   rG   rH   rank_to_intbytenrQ   vocab_bpe_contents
bpe_merges	bpe_ranksfirstsecondro   encoder_jsonencoder_json_loadedra   rd   s                  @@r   data_gym_to_mergeable_bpe_ranksr|   V   s    YX%++XXXO@@@@@	A4[[  O##""1%%%34!#dQh--0FA4'''' *..IIPPRRaa<N<T<TUY<Z<Z[\]_[_<`aaaJ> > > > > > GF9_+E+EFFFIIA#  vFG	//%((??6+B+BBC	QKKK
 ::./@BSTTUULRRRR\=O=O=Q=QRRR,d333.555+++++r#   rw   tiktoken_bpe_fileNonec                   	 dd l }n"# t          $ r}t          d          |d }~ww xY w|                    |d          5 }t          |                                 d           D ]T\  }}|                    t          j        |          dz   t          |          	                                z   dz              U	 d d d            d S # 1 swxY w Y   d S )Nr   r	   r+   c                    | d         S )NrW   rU   )xs    r   <lambda>z#dump_tiktoken_bpe.<locals>.<lambda>   s
    1Q4 r#   )key       
)
r   r   r   sortedrq   r<   base64	b64encoder   r3   )rw   r}   r   r   r   tokenranks          r   dump_tiktoken_bper      sK      ]
 
	 
		,d	3	3 Qq!)//"3"3HHH 	Q 	QKE4GGF$U++d2SYY5E5E5G5GG%OPPPP	QQ Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Qs"    
&!&A:CCCc                &   t          | |          }i }|                                D ]i}|s	 |                                \  }}t          |          |t	          j        |          <   B# t          $ r}t          d|d|            |d }~ww xY w|S )NzError parsing line z in )rD   
splitlinesrZ   intr   	b64decode	Exceptionr8   )r}   r   rB   retliner   r   r   s           r   load_tiktoken_bper      s     1=AAH
C##%% [ [ 		[**,,KE4+.t99C ''(( 	[ 	[ 	[R4RR?PRRSSYZZ	[Js   ;A))
B3B		B)r   r   r   r   )r   r   r   r   r   r   r   )r   r   r   r$   r   r   )NN)
rE   r   rF   r   rG   r$   rH   r$   r   rI   )rw   rI   r}   r   r   r~   )r}   r   r   r$   r   rI   )
__future__r   r   r   r,   r   r"   rD   r|   r   r   rU   r#   r   <module>r      s    " " " " " "   				   &( ( ( (
3 3 3 3 3r "&$(	- - - - -`	Q 	Q 	Q 	Q      r#   