o
    ì6Wj�  ã                   @  s  U d dl mZ d dlZd dlmZ d dlmZ ejdkr#d dlm	Z	 nzd dl
m	Z	 W n ey8   dd„ Z	Y nw d	d
lmZmZmZmZmZmZmZmZmZmZmZmZmZ d	dlmZmZmZmZmZm Z m!Z! eeB eB eB eB Z"de#d< e	G dd„ dƒƒZ$edd�d>dd„ƒZ%dd„ e&dƒD ƒZ'de#d< G dd„ dƒZ(e	G dd„ de(ƒƒZ)e	G dd „ d e(ƒƒZ*e	G d!d"„ d"e(ƒƒZ+e	G d#d$„ d$e(ƒƒZ,e	G d%d&„ d&e(ƒƒZ-e	G d'd(„ d(e(ƒƒZ.e	G d)d*„ d*e(ƒƒZ/e	G d+d,„ d,e(ƒƒZ0e	G d-d.„ d.e(ƒƒZ1ed/d�d?d4d5„ƒZ2	7d@dAd<d=„Z3dS )Bé    )ÚannotationsN)Ú	lru_cache)Ú	getLogger)é   é   )Úfinalc                 C  s   | S )N© )Úclsr   r   úd/home/esfera/Documents/content_generation/venv/lib/python3.10/site-packages/charset_normalizer/md.pyr      s   r   é   )ÚCOMMON_CJK_CHARACTERSÚCOMMON_SAFE_ASCII_CHARACTERSÚTRACEÚUNICODE_SECONDARY_RANGE_KEYWORDÚ_ACCENTUATEDÚ_ARABICÚ_ARABIC_ISOLATED_FORMÚ_CJKÚ_HANGULÚ	_HIRAGANAÚ	_KATAKANAÚ_LATINÚ_THAI)Ú_character_flagsÚis_emoticonÚis_punctuationÚis_separatorÚ	is_symbolÚremove_accentÚunicode_rangeÚintÚ_GLYPH_MASKc                   @  sÐ   e Zd ZU dZdZded< ded< ded< ded< ded	< ded
< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< d"dd „Zd!S )#ÚCharInfoz>Pre-computed character properties shared across all detectors.)Ú	characterÚ	printableÚalphaÚupperÚlowerÚspaceÚdigitÚis_asciiÚcase_variableÚflagsÚaccentuatedÚlatinÚis_cjkÚ	is_arabicÚis_glyphÚpunctÚsymÚrangeÚsepÚemoticonÚsafeÚ
common_cjkÚstrr#   Úboolr$   r%   r&   r'   r(   r)   r*   r+   r    r,   r-   r.   r/   r0   r1   r2   r3   ú
str | Noner4   r5   r6   r7   r8   ÚreturnÚNonec                 C  s0  || _ t|ƒ}|dk �rd| _d| _d| _d| _d| _d|  kr%dkrJn n#d| _d| _d| _	d| _
d| _d| _d| _t| _d| _d| _d| _�n3d|  krTdkryn n#d| _d| _d| _	d| _
d| _d| _d| _t| _d| _d| _d| _�nd|  krƒd	kr§n n"d| _d| _d| _	d| _
d| _d| _d| _d
| _d| _d| _d| _nÖ|dks·d|  krµdkrÛn n$d| _d| _d| _	d| _
d| _|dk| _d| _d
| _d| _d| _d| _n¢| ¡ | _d| _d| _d| _	d| _
d| _d| _d
| _d| _| jrÿt|ƒnd| _| j�r
t|ƒnd| _nod| _| ¡ | _| ¡ | _| ¡ | _| ¡ | _	| ¡ | _
| ¡ | _| j	| jk| _| j�r?t|ƒ}nd
}|| _t|t@ ƒ| _t|t@ ƒ| _t|t@ ƒ| _t|t@ ƒ| _t|t @ ƒ| _| j�rot|ƒnd| _| j�rzt|ƒnd| _t!|ƒ| _"t#|ƒ| _$t%|ƒ| _&|t'v | _(|t)v | _*dS )zkCompute all properties for *character* (built once per codepoint,
        every branch assigns every slot).é€   TFéA   éZ   éa   éz   é0   é9   r   é    é	   é   N)+r#   Úordr*   r-   r/   r0   r1   r%   r&   r'   r(   r)   r$   r+   r   r,   r.   r2   r3   Úisprintabler   r   ÚisalphaÚisupperÚislowerÚisspaceÚisdigitr   r:   r   r   r   r!   r   r4   r   r5   r   r6   r   r7   r   r8   )Úselfr#   Úor,   r   r   r
   Ú__init__c   s¶   


 












zCharInfo.__init__N)r#   r9   r<   r=   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú	__slots__Ú__annotations__rQ   r   r   r   r
   r"   /   s4   
 r"   )Úmaxsizer#   r9   r<   c                 C  s   t | ƒS )zBBuild (once per codepoint) and cache the CharInfo for *character*.)r"   )r#   r   r   r
   Ú
_char_infoÙ   s   rY   c                 C  s   g | ]}t t|ƒƒ‘qS r   )r"   Úchr)Ú.0Ú
_codepointr   r   r
   Ú
<listcomp>à   s    ÿr]   r>   zlist[CharInfo]Ú_ASCII_CHAR_INFOc                   @  s6   e Zd ZdZdZdd	d
„Zddd„Zeddd„ƒZdS )ÚMessDetectorPluginzy
    Base abstract class used for mess detection plugins.
    All detectors MUST extend and implement given methods.
    r   r#   r9   Úinfor"   r<   r=   c                 C  ó   t ‚)z‰
        The main routine to be executed upon character.
        Insert the logic in witch the text would be considered chaotic.
        ©ÚNotImplementedError©rO   r#   r`   r   r   r
   Ú	feed_infoí   s   zMessDetectorPlugin.feed_infoc                 C  ra   )zB
        Permit to reset the plugin to the initial state.
        rb   ©rO   r   r   r
   Úresetô   s   zMessDetectorPlugin.resetÚfloatc                 C  ra   )z…
        Compute the chaos ratio based on what your feed() has seen.
        Must NOT be lower than 0.; No restriction gt 0.
        rb   rf   r   r   r
   Úratioú   s   zMessDetectorPlugin.ratioN©r#   r9   r`   r"   r<   r=   ©r<   r=   ©r<   rh   )	rR   rS   rT   rU   rV   re   rg   Úpropertyri   r   r   r   r
   r_   å   s    

r_   c                   @  ó<   e Zd ZdZddd„Zdd
d„Zddd„Zeddd„ƒZdS )Ú TooManySymbolOrPunctuationPlugin©Ú_punctuation_countÚ_symbol_countÚ_character_countÚ_last_printable_charÚ_frenzy_symbol_in_wordr<   r=   c                 C  s"   d| _ d| _d| _d | _d| _d S ©Nr   Frp   rf   r   r   r
   rQ     s
   
z)TooManySymbolOrPunctuationPlugin.__init__r#   r9   r`   r"   c                 C  s^   |  j d7  _ || jkr*|js*|jr|  jd7  _n|js*|jr*|js*|  jd7  _|| _dS )ú1Optimized feed using pre-computed character info.r   é   N)	rs   rt   r7   r2   rq   r)   r3   r6   rr   rd   r   r   r
   re     s   
z*TooManySymbolOrPunctuationPlugin.feed_infoc                 C  s   d| _ d| _d| _d S ©Nr   )rq   rs   rr   rf   r   r   r
   rg   !  s   
z&TooManySymbolOrPunctuationPlugin.resetrh   c                 C  s0   | j dkrdS | j| j | j  }|dkr|S dS )Nr   ç        ç333333Ó?)rs   rq   rr   )rO   Úratio_of_punctuationr   r   r
   ri   &  s   

þz&TooManySymbolOrPunctuationPlugin.ratioNrk   rj   rl   ©	rR   rS   rT   rV   rQ   re   rg   rm   ri   r   r   r   r
   ro     s    


ro   c                   @  rn   )ÚTooManyAccentuatedPlugin©rs   Ú_accentuated_countr<   r=   c                 C  ó   d| _ d| _d S ry   r   rf   r   r   r
   rQ   6  ó   
z!TooManyAccentuatedPlugin.__init__r#   r9   r`   r"   c                 C  s*   |  j d7  _ |jr|  jd7  _dS dS ©rw   r   N)rs   r-   r€   rd   r   r   r
   re   :  ó   ÿz"TooManyAccentuatedPlugin.feed_infoc                 C  r�   ry   r   rf   r   r   r
   rg   A  r‚   zTooManyAccentuatedPlugin.resetrh   c                 C  s*   | j dk rdS | j| j  }|dkr|S dS )Nr   rz   gffffffÖ?r   )rO   Úratio_of_accentuationr   r   r
   ri   E  s   
zTooManyAccentuatedPlugin.ratioNrk   rj   rl   r}   r   r   r   r
   r~   2  s    


r~   c                   @  rn   )ÚUnprintablePlugin©Ú_unprintable_countrs   r<   r=   c                 C  r�   ry   r‡   rf   r   r   r
   rQ   R  r‚   zUnprintablePlugin.__init__r#   r9   r`   r"   c                 C  s<   |j s|js|dkr|dkr|  jd7  _|  jd7  _dS )rw   úu   ï»¿r   N)r(   r$   rˆ   rs   rd   r   r   r
   re   V  s   ÿþzUnprintablePlugin.feed_infoc                 C  s
   d| _ d S ry   )rˆ   rf   r   r   r
   rg   a  s   
zUnprintablePlugin.resetrh   c                 C  ó   | j dkrdS | jd | j  S )Nr   rz   r   )rs   rˆ   rf   r   r   r
   ri   d  ó   
zUnprintablePlugin.ratioNrk   rj   rl   r}   r   r   r   r
   r†   N  s    


r†   c                   @  rn   )ÚSuspiciousDuplicateAccentPlugin©Ú_successive_countrs   Ú_last_latin_characterÚ_last_was_accentuatedr<   r=   c                 C  ó   d| _ d| _d | _d| _d S rv   r�   rf   r   r   r
   rQ   u  s   
z(SuspiciousDuplicateAccentPlugin.__init__r#   r9   r`   r"   c                 C  st   |  j d7  _ | jdur1|jr1| jr1|jr!| j ¡ r!|  jd7  _t|ƒt| jƒkr1|  jd7  _|| _|j| _dS rƒ   )rs   r�   r-   r�   r&   rK   rŽ   r   rd   r   r   r
   re   |  s   
ÿþz)SuspiciousDuplicateAccentPlugin.feed_infoc                 C  r‘   rv   r�   rf   r   r   r
   rg   ‹  ó   
z%SuspiciousDuplicateAccentPlugin.resetrh   c                 C  rŠ   )Nr   rz   rx   )rs   rŽ   rf   r   r   r
   ri   ‘  r‹   z%SuspiciousDuplicateAccentPlugin.ratioNrk   rj   rl   r}   r   r   r   r
   rŒ   l  s    


rŒ   c                   @  rn   )ÚSuspiciousRange©Ú"_suspicious_successive_range_countrs   Ú_last_printable_seenÚ_last_printable_ranger<   r=   c                 C  ó   d| _ d| _d | _d | _d S ry   r”   rf   r   r   r
   rQ   ¢  r’   zSuspiciousRange.__init__r#   r9   r`   r"   c                 C  s�   |  j d7  _ |js|js|jrd| _d| _dS | jdu r&|| _|j| _dS | j}|j}||ks4|du r@t||ƒr@|  jd7  _|| _|| _dS rƒ   )	rs   r(   r2   r7   r–   r—   r4   Ú is_suspiciously_successive_ranger•   )rO   r#   r`   Úunicode_range_aÚunicode_range_br   r   r
   re   ¨  s    


zSuspiciousRange.feed_infoc                 C  r˜   ry   )rs   r•   r–   r—   rf   r   r   r
   rg   Á  r’   zSuspiciousRange.resetrh   c                 C  s"   | j dkrdS | jd | j  }|S )NrG   rz   rx   )rs   r•   )rO   Úratio_of_suspicious_range_usager   r   r
   ri   Ç  s   
þzSuspiciousRange.ratioNrk   rj   rl   r}   r   r   r   r
   r“   ™  s    


r“   c                   @  rn   )ÚSuperWeirdWordPlugin©Ú_word_countÚ_bad_word_countÚ_foreign_long_countÚ_is_current_word_badÚ_foreign_long_watchrs   Ú_bad_character_countÚ_buffer_lengthÚ_buffer_last_charÚ_buffer_last_char_accentuatedÚ_buffer_accent_countÚ_buffer_glyph_countÚ_buffer_upper_countÚ_buffer_first_lowerÚ_buffer_has_non_asciir<   r=   c                 C  s^   d| _ d| _d| _d| _d| _d| _d| _d| _d | _d| _	d| _
d| _d| _d| _d| _d S rv   rž   rf   r   r   r
   rQ   ç  s   
zSuperWeirdWordPlugin.__init__r#   r9   r`   r"   c                 C  sd  |j rO| jdkr|j| _|  jd7  _|| _|jr |  jd7  _|js&d| _|j	| _
|j	r4|  jd7  _| jsC|jr=|j	rC|jsCd| _|jrM|  jd7  _dS | jsTdS |js^|js^|j�r|  jd7  _| j}|  j|7  _|dkr½| j| dkr~d| _n?| j
r–| j ¡ r–| j|kr–|  jd7  _d| _n'| jdkr¦d| _|  jd7  _n| jr½| jr½| j|d kr½|  jd7  _d| _|dkrÜ| jrÜ| jdkoÏ| j| dk}|sÜ|  jd7  _d| _| jrð|  jd7  _|  j|7  _d	| _d	| _d| _d| _d	| _
d| _d| _d| _d	| _d	| _dS |d
v�r,|j�s.|j�r0d| _|  jd7  _|| _d	| _
dS dS dS dS )rw   r   r   TNé   ç      à?é   r{   F>   Ú_ú-ú<ú=ú>ú|ú~)r%   r¥   r'   r«   r¦   r&   rª   r*   r¬   r-   r§   r¨   r£   r.   r1   r©   r(   r2   r5   rŸ   rs   r¢   rK   r¡   r    r¤   r)   r3   )rO   r#   r`   Úbuffer_lengthÚprobable_camel_casedr   r   r
   re   û  s    
ÿþþýÿþ

ÿþ
þ

ÿþ
ùzSuperWeirdWordPlugin.feed_infoc                 C  s^   d| _ d | _d| _d| _d| _d| _d| _d| _d| _d| _	d| _
d| _d| _d| _d| _d S rv   )r¥   r¦   r§   r¢   r£   r    rŸ   rs   r¤   r¡   r¨   r©   rª   r«   r¬   rf   r   r   r
   rg   V  s   
zSuperWeirdWordPlugin.resetrh   c                 C  s$   | j dkr| jdkrdS | j| j S )Né
   r   rz   )rŸ   r¡   r¤   rs   rf   r   r   r
   ri   g  s   zSuperWeirdWordPlugin.ratioNrk   rj   rl   r}   r   r   r   r
   r�   Ó  s    


[r�   c                   @  s@   e Zd ZdZdZddd„Zddd„Zddd„Zeddd„ƒZ	dS )ÚCjkUncommonPluginz<
    Detect messy CJK text that probably means nothing.
    ©rs   Ú_uncommon_countr<   r=   c                 C  r�   ry   r»   rf   r   r   r
   rQ   w  r‚   zCjkUncommonPlugin.__init__r#   r9   r`   r"   c                 C  s*   |  j d7  _ |js|  jd7  _dS dS rƒ   )rs   r8   r¼   rd   r   r   r
   re   {  r„   zCjkUncommonPlugin.feed_infoc                 C  r�   ry   r»   rf   r   r   r
   rg   ‚  r‚   zCjkUncommonPlugin.resetrh   c                 C  s.   | j dk rdS | j| j  }|dkr|d S dS )Nr   rz   r®   r¹   r»   )rO   Úuncommon_form_usager   r   r
   ri   †  s   
zCjkUncommonPlugin.ratioNrk   rj   rl   )
rR   rS   rT   rU   rV   rQ   re   rg   rm   ri   r   r   r   r
   rº   o  s    


rº   c                   @  rn   )ÚArchaicUpperLowerPlugin©	Ú_bufÚ_character_count_since_last_sepÚ_successive_upper_lower_countÚ#_successive_upper_lower_count_finalrs   Ú_last_alpha_seenÚ_last_alpha_seen_upperÚ_last_alpha_seen_lowerÚ_current_ascii_onlyr<   r=   c                 C  s:   d| _ d| _d| _d| _d| _d | _d| _d| _d| _d S )NFr   Tr¿   rf   r   r   r
   rQ      s   
z ArchaicUpperLowerPlugin.__init__r#   r9   r`   r"   c                 C  s
  |j o|j}| }|r;| jdkr;| jdkr#|js#| js#|  j| j7  _d| _d| _d| _d| _|  j	d7  _	d| _dS | jrD|j
sDd| _| jdurj|jrO| jsU|jrg| jrg| jrc|  jd7  _d| _nd| _nd| _|  j	d7  _	|  jd7  _|| _|j| _|j| _dS )rw   r   é@   NFr   Trx   )r%   r+   rÁ   r)   rÇ   rÃ   rÂ   rÄ   rÀ   rs   r*   r&   rÆ   r'   rÅ   )rO   r#   r`   Úis_concernedÚ	chunk_sepr   r   r
   re   ¯  sH   
ÿþÿ
ÿÿz!ArchaicUpperLowerPlugin.feed_infoc                 C  s:   d| _ d| _d| _d| _d | _d| _d| _d| _d| _d S )Nr   FT)	rs   rÁ   rÂ   rÃ   rÄ   rÅ   rÆ   rÀ   rÇ   rf   r   r   r
   rg   Ü  s   
zArchaicUpperLowerPlugin.resetrh   c                 C  s   | j dkrdS | j| j  S )Nr   rz   )rs   rÃ   rf   r   r   r
   ri   ç  s   
zArchaicUpperLowerPlugin.ratioNrk   rj   rl   r}   r   r   r   r
   r¾   ’  s    


-r¾   c                   @  s<   e Zd ZdZddd„Zddd„Zddd„Zeddd„ƒZdS )ÚArabicIsolatedFormPlugin©rs   Ú_isolated_form_countr<   r=   c                 C  r�   ry   rÌ   rf   r   r   r
   rQ   ó  r‚   z!ArabicIsolatedFormPlugin.__init__c                 C  r�   ry   rÌ   rf   r   r   r
   rg   ÷  r‚   zArabicIsolatedFormPlugin.resetr#   r9   r`   r"   c                 C  s.   |  j d7  _ |jt@ r|  jd7  _dS dS rƒ   )rs   r,   r   rÍ   rd   r   r   r
   re   û  s   
ÿz"ArabicIsolatedFormPlugin.feed_inforh   c                 C  s   | j dk rdS | j| j  }|S )Nr   rz   rÌ   )rO   Úisolated_form_usager   r   r
   ri     s   
zArabicIsolatedFormPlugin.ratioNrk   rj   rl   )	rR   rS   rT   rV   rQ   rg   re   rm   ri   r   r   r   r
   rË   ï  s    


rË   é   rš   r;   r›   r:   c                 C  sv  | du s|du r
dS | |krdS d| v rd|v rdS d| v s"d|v r$dS d| v s,d|v r6d| v s4d|v r6dS |   d¡|  d¡}}|D ]}|tv rJqC||v rQ dS qC| dv |dv }}|s_|rid	| v sgd	|v ridS |ro|rodS d
| v swd
|v r‹d	| v sd	|v r�dS | dks‰|dkr‹dS d	| v s›d	|v s›| dv r¹|dv r¹d| v s£d|v r¥dS d| v s­d|v r¯dS | dks·|dkr¹dS dS )za
    Determine if two Unicode range seen next to each other can be considered as suspicious.
    NTFÚLatinÚ	EmoticonsÚ	Combiningú )ÚHiraganaÚKatakanaÚCJKÚHangulzBasic Latin)rÕ   rÔ   ÚPunctuationÚForms)Úsplitr   )rš   r›   Úkeywords_range_aÚkeywords_range_bÚelÚrange_a_jp_charsÚrange_b_jp_charsr   r   r
   r™     sZ   þÿÿúr™   çš™™™™™É?FÚdecoded_sequenceÚmaximum_thresholdrh   Údebugc           "   	   C  sÊ  t | ƒ}|dk rd}n	|dk rd}nd}|  ¡ }t}t}tƒ }tƒ }	tƒ }
tƒ }tƒ }t	ƒ }t
ƒ }tƒ }tƒ }|j}|	j}|
j}|j}|j}|j}|j}|j}|j}td||ƒD ]†}| ||| … D ]Z}t|ƒ}|dk rq|| }n||ƒ}|||ƒ |||ƒ |rŠ|jr‰|||ƒ qb|||ƒ |jrœ|||ƒ |||ƒ |jr¼|||ƒ |jr¬|||ƒ |jr´|||ƒ |jr¼|||ƒ qb|j|	j |
j |j |j |j |j |j |j }||krÞ n1qX|d }|d|ƒ |sï|d|ƒ |d|ƒ |j|	j |
j |j |j |j |j |j |j }|�r`td	ƒ} |  td
|› d|› d|› �¡ |dk�rD|  td| dd… › �¡ |  td| dd… › �¡ ||	|
||||||f	D ]}!|  t|!j› d|!j› �¡ �qOt|dƒS )zw
    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
    iÿ  rE   rÏ   rÈ   r>   r   r¹   Ú
Úcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=é   zStarting with: NzEnding with: iðÿÿÿz: r   )ÚlenÚisasciir^   rY   ro   r~   r†   rŒ   r“   r�   rº   r¾   rË   re   r4   rH   r$   r%   r.   r/   r0   ri   r   Úlogr   Ú	__class__Úround)"rá   râ   rã   Úseq_lenÚstepÚis_pure_asciiÚ
ascii_infoÚ	char_infoÚd_spÚd_taÚd_upÚd_sdaÚd_srÚd_swÚd_cuÚd_auÚd_aiÚ	d_sp_feedÚ	d_ta_feedÚ	d_up_feedÚ
d_sda_feedÚ	d_sr_feedÚ	d_sw_feedÚ	d_cu_feedÚ	d_au_feedÚ	d_ai_feedÚblock_startr#   Ú	codepointr`   Úmean_mess_ratioÚnl_infoÚloggerÚdtr   r   r
   Ú
mess_ratioV  sä   	










€ÿþýüûúùøÿÿ


ÿþýüûúùøÿÿÿþþ

r	  )r#   r9   r<   r"   )rš   r;   r›   r;   r<   r:   )rà   F)rá   r9   râ   rh   rã   r:   r<   rh   )4Ú
__future__r   ÚsysÚ	functoolsr   Úloggingr   Úversion_infoÚtypingr   Útyping_extensionsÚImportErrorÚconstantr   r   r   r   r   r   r   r   r   r   r   r   r   Úutilsr   r   r   r   r   r   r   r!   rW   r"   rY   r4   r^   r_   ro   r~   r†   rŒ   r“   r�   rº   r¾   rË   r™   r	  r   r   r   r
   Ú<module>   s\    
þ<$ *ÿ.,9 "\Jÿ