/ whisper-webui.git / app / env / Lib / site-packages / tokenizers / __pycache__ / __init__.cpython-310.pyc

o �J�h� �@s,
dd
lm
Z

ddlmZmZmZ
eeefZeZ eeeeefZ ee ee e fee fZ ee ee e fee fZ ee e fZ eeefZ Gdd�de
�ZGdd�de
�ZGdd�de
�Zd d lmZmZmZmZmZmZmZmZmZmZmZmZmZm Z
d dl!m"Z"m#Z#m$Z$m%Z%m&Z&
dS) �)
�Enum)�List�Tuple�Unionc
@�eZ
dZd
ZdZdS)�OffsetReferential�original� normalizedN)�__name__� __module__�__qualname__ZORIGINALZ NORMALIZED�r r �QC:\pinokio\api\whisper-webui.git\app\env\lib\site-packages\tokenizers\__init__.pyr<�

rc
@r)� OffsetType�byte�charN)r rr�BYTE�CHARr r r rrArrc
@s eZ
dZd
ZdZdZdZdZdS)�SplitDelimiterBehavior�removed�isolatedZmerged_with_previousZmerged_with_next� contiguousN)r rrZREMOVEDZISOLATEDZMERGED_WITH_PREVIOUSZMERGED_WITH_NEXT� CONTIGUOUSr r r rrFs




r�
)� AddedToken�Encoding�NormalizedString�PreTokenizedString�Regex�Token� Tokenizer�decoders�models�normalizers�pre_tokenizers� processors�trainers�__version__)�BertWordPieceTokenizer�ByteLevelBPETokenizer�CharBPETokenizer�SentencePieceBPETokenizer�SentencePieceUnigramTokenizerN)'�enumr�typingrrr�intZOffsets�strZTextInputSequenceZPreTokenizedInputSequenceZTextEncodeInputZPreTokenizedEncodeInputZ InputSequenceZEncodeInputrrr� tokenizersrrrrrr r!r"r#r$r%r&r'r(�implementationsr)r*r+r,r-r r r r�<module>
s<





�� 


�� 

@