
    g                    t    d dl mZ d dlZd dlZd dlZd	dZd
dZdddZ	 	 d	 	 	 	 	 	 	 	 	 ddZddZ	dddZ
y)    )annotationsNc                \   | j                  d      sB| j                  d      s1	 dd l}|j                  | d      5 }|j	                         cd d d        S dd l}|j                  |       }|j                          |j                  S # t        $ r}t        d      |d }~ww xY w# 1 sw Y   WxY w)Nzhttp://zhttps://r   Oblobfile is not installed. Please install it by running `pip install blobfile`.rb)	
startswithblobfileImportErrorBlobFilereadrequestsgetraise_for_statuscontent)blobpathr   efr   resps         c/var/www/python.lazyprojects.co.uk/rdoDiscordBot/venv/lib/python3.12/site-packages/tiktoken/load.py	read_filer      s    y)(2E2Ej2Q	
 x.!668 /. <<!D<<  	a	 /.s"   B B"	BBB"B+c                R    t        j                  |       j                         }||k(  S N)hashlibsha256	hexdigest)dataexpected_hashactual_hashs      r   
check_hashr      s$    ..&002K-''    c                ^   d}dt         j                  v rt         j                  d   }nZdt         j                  v rt         j                  d   }n4dd l}t         j                  j	                  |j                         d      }d}|dk(  rt        |       S t        j                  | j                               j                         }t         j                  j	                  ||      }t         j                  j                  |      rKt        |d      5 }|j                         }d d d        |t        |      rS 	 t        j                  |       t        |       }	|rt        |	|      st#        d	|  d
| d      dd l}
	 t        j&                  |d       |dz   t)        |
j+                               z   dz   }t        |d      5 }|j-                  |	       d d d        t        j.                  ||       |	S # 1 sw Y   xY w# t         $ r Y w xY w# 1 sw Y   <xY w# t         $ r |r Y |	S w xY w)NTTIKTOKEN_CACHE_DIRDATA_GYM_CACHE_DIRr   zdata-gym-cacheF r   z'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)exist_ok.z.tmpwb)osenvirontempfilepathjoin
gettempdirr   r   sha1encoder   existsopenr   r   removeOSError
ValueErroruuidmakedirsstruuid4writerename)r   r   user_specified_cache	cache_dirr)   	cache_key
cache_pathr   r   contentsr4   tmp_filenames               r   read_file_cachedr@       s   rzz)JJ34			+JJ34	GGLL!4!4!68HI	$B""X__./99;Ii3J	ww~~j!*d#q668D $ Jt]$CK	IIj! "HZ-@5hZ{=/ ZH I
 	

 	
I-!C'#djjl*;;fD,%GGH &
		,
+ O? $#  		 &%     OsI   G59H >AH HH 5G>	HHHH H,+H,c                   t        d      D cg c]-  }t        |      j                         st        |      dk7  s,|/ }}|D ci c]  }t        |      | c}d}t        d      D ].  }||vs|j                  |       |t        d|z         <   |dz  }0 t	        |      dk(  sJ t        | |      j                         }|j                  d      dd D cg c]  }t        |j                                }	}d
fd}
t        |      D ci c]  \  }}t        |g      | }}}t	        |      }|	D ]  \  }}|| |
|       |
|      z   <   |dz  }  dd l}|j                  t        ||            }|j                         D ci c]  \  }} |
|      | }}}|j                  dd        |j                  d	d        ||k(  sJ |S c c}w c c}w c c}w c c}}w c c}}w )N    r      
c                ,    t        fd| D              S )Nc              3  (   K   | ]	  }|     y wr    ).0bdata_gym_byte_to_bytes     r   	<genexpr>zKdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>m   s     =u!*1-us   )bytes)valuerL   s    r   decode_data_gymz8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gyml   s    =u===r   s   <|endoftext|>s   <|startoftext|>)rO   r6   returnrN   )rangechrisprintableappendlenr@   decodesplittuple	enumeraterN   jsonloadsitemspop)vocab_bpe_fileencoder_json_filevocab_bpe_hashencoder_json_hashrK   rank_to_intbytenvocab_bpe_contents	merge_str
bpe_mergesrP   i	bpe_ranksfirstsecondr[   encoder_jsonkvencoder_json_loadedrL   s                       @r   data_gym_to_mergeable_bpe_ranksrp   V   s    #(+X+QQ1C1C1E#a&TW-q+OX0?@1SVQY@	A4[O#""1%34!#dQh-0FA	 
 4''' *..IPPR<N<T<TUY<Z[\]_<`a<`y%	)*<`Ja> ,5_+EF+E41asQ+EIFIA#vFG	/%(?6+BBC	Q $ 
 ::./@BSTUL=I=O=O=QR=QTQ?1-q0=QR,d3.5++++M Y@ b G Ss'   GGGG GGG#c                h   	 dd l }|j                  |d      5 }t        | j	                         d       D ]I  \  }}|j                  t        j                  |      dz   t        |      j                         z   dz          K 	 d d d        y # t        $ r}t        d      |d }~ww xY w# 1 sw Y   y xY w)Nr   r   r&   c                    | d   S )NrD   rI   )xs    r   <lambda>z#dump_tiktoken_bpe.<locals>.<lambda>   s    1Q4r   )key       
)
r   r	   r
   sortedr]   r8   base64	b64encoder6   r.   )ri   tiktoken_bpe_filer   r   r   tokenranks          r   dump_tiktoken_bper~      s    
 
		,d	3q!)//"3HKE4GGF$$U+d2SY5E5E5GG%OP I 
4	3	  ]
	 
4	3s#   B A)B(	B%B  B%(B1c                   t        | |      }i }|j                         D ]:  }|s	 |j                         \  }}t        |      |t	        j
                  |      <   < |S # t        $ r}t        d|d|        |d }~ww xY w)NzError parsing line z in )r@   
splitlinesrX   intry   	b64decode	Exceptionr3   )r{   r   r>   retliner|   r}   r   s           r   load_tiktoken_bper      s     1=AH
C##%	[**,KE4+.t9C  '( & J  	[24($?P>QRSYZZ	[s   4A	A>'A99A>)r   r6   rQ   rN   )r   rN   r   r6   rQ   boolr   )r   r6   r   
str | NonerQ   rN   )NN)
r_   r6   r`   r6   ra   r   rb   r   rQ   dict[bytes, int])ri   r   r{   r6   rQ   None)r{   r6   r   r   rQ   r   )
__future__r   ry   r   r'   r   r   r@   rp   r~   r   rI   r   r   <module>r      sh    "   	&(
3r "&$(	--- - "	-
 -`	Qr   