o
    Õ­jév  ã                   @  sò  U d dl mZ d dlZd dlmZ d dlmZ ejdkr#d dlm	Z	 nzd dl
m	Z	 W n ey8   dd„ Z	Y nw d	d
lmZmZmZmZmZmZmZmZmZmZmZmZmZ d	dlmZmZmZmZmZm Z m!Z! eeB eB eB eB Z"de#d< e	G dd„ dƒƒZ$G dd„ dƒZ%e	G dd„ de%ƒƒZ&e	G dd„ de%ƒƒZ'e	G dd„ de%ƒƒZ(e	G dd„ de%ƒƒZ)e	G dd„ de%ƒƒZ*e	G dd„ de%ƒƒZ+e	G dd„ de%ƒƒZ,e	G d d!„ d!e%ƒƒZ-e	G d"d#„ d#e%ƒƒZ.ed$d%�d7d+d,„ƒZ/ed-d%�	/d8d9d5d6„ƒZ0dS ):é    )ÚannotationsN)Ú	lru_cache)Ú	getLogger)é   é   )Úfinalc                 C  s   | S )N© )Úclsr   r   úR/var/www/html/CropPilot/venv/lib/python3.10/site-packages/charset_normalizer/md.pyr      s   r   é   )ÚCOMMON_CJK_CHARACTERSÚCOMMON_SAFE_ASCII_CHARACTERSÚTRACEÚUNICODE_SECONDARY_RANGE_KEYWORDÚ_ACCENTUATEDÚ_ARABICÚ_ARABIC_ISOLATED_FORMÚ_CJKÚ_HANGULÚ	_HIRAGANAÚ	_KATAKANAÚ_LATINÚ_THAI)Ú_character_flagsÚis_emoticonÚis_punctuationÚis_separatorÚ	is_symbolÚremove_accentÚunicode_rangeÚintÚ_GLYPH_MASKc                   @  s(   e Zd ZdZdZddd„Zdd	d
„ZdS )ÚCharInfou{  Pre-computed character properties shared across all detectors.

    Instantiated once and reused via :meth:`update` on every character
    in the hot loop so that redundant calls to str methods
    (``isalpha``, ``isupper``, â€¦) and cached utility functions
    (``_character_flags``, ``is_punctuation``, â€¦) are avoided when
    several plugins need the same information.
    ©Ú	characterÚ	printableÚalphaÚupperÚlowerÚspaceÚdigitÚis_asciiÚcase_variableÚflagsÚaccentuatedÚlatinÚis_cjkÚ	is_arabicÚis_glyphÚpunctÚsymÚreturnÚNonec                 C  sj   d| _ d| _d| _d| _d| _d| _d| _d| _d| _d| _	d| _
d| _d| _d| _d| _d| _d| _d S )NÚ Fr   r#   ©Úselfr   r   r
   Ú__init__N   s"   
zCharInfo.__init__r$   Ústrc                 C  s  || _ t|ƒ}|dk �rd| _d| _d| _d| _d| _d|  kr%dkrJn n#d| _d| _d| _	d| _
d| _d| _d| _t| _d| _d| _d| _dS d|  krTdkryn n#d| _d| _d| _	d| _
d| _d| _d| _t| _d| _d| _d| _dS d|  krƒd	kr¨n n#d| _d| _d| _	d| _
d| _d| _d| _d
| _d| _d| _d| _dS |dks¸d|  kr¶dkrÝn n%d| _d| _d| _	d| _
d| _|dk| _d| _d
| _d| _d| _d| _dS | ¡ | _d| _d| _d| _	d| _
d| _d| _d
| _d| _| j�rt|ƒnd| _| j�rt|ƒnd| _dS d| _| ¡ | _| ¡ | _| ¡ | _| ¡ | _	| ¡ | _
| ¡ | _| j	| jk| _| j�rCt|ƒ}nd
}|| _t|t@ ƒ| _t|t@ ƒ| _t|t@ ƒ| _t|t@ ƒ| _t|t @ ƒ| _| j�rst|ƒnd| _| j�r~t|ƒnd| _dS )zBUpdate all properties for *character* (called once per character).é€   TFéA   éZ   éa   éz   é0   é9   r   é    é	   é   N)!r$   Úordr+   r.   r0   r1   r2   r&   r'   r(   r)   r*   r%   r,   r   r-   r/   r3   r4   Úisprintabler   r   ÚisalphaÚisupperÚislowerÚisspaceÚisdigitr   Úboolr   r   r   r!   )r9   r$   Úor-   r   r   r
   Úupdatea   s¬   



 









zCharInfo.updateN©r5   r6   )r$   r;   r5   r6   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú	__slots__r:   rO   r   r   r   r
   r"   /   s
    	
r"   c                   @  s6   e Zd ZdZdZdd	d
„Zddd„Zeddd„ƒZdS )ÚMessDetectorPluginzy
    Base abstract class used for mess detection plugins.
    All detectors MUST extend and implement given methods.
    r   r$   r;   Úinfor"   r5   r6   c                 C  ó   t ‚)z‰
        The main routine to be executed upon character.
        Insert the logic in witch the text would be considered chaotic.
        ©ÚNotImplementedError©r9   r$   rW   r   r   r
   Ú	feed_infoÖ   s   zMessDetectorPlugin.feed_infoc                 C  rX   )zB
        Permit to reset the plugin to the initial state.
        rY   r8   r   r   r
   ÚresetÝ   s   zMessDetectorPlugin.resetÚfloatc                 C  rX   )z…
        Compute the chaos ratio based on what your feed() has seen.
        Must NOT be lower than 0.; No restriction gt 0.
        rY   r8   r   r   r
   Úratioã   s   zMessDetectorPlugin.ratioN©r$   r;   rW   r"   r5   r6   rP   ©r5   r^   )	rQ   rR   rS   rT   rU   r\   r]   Úpropertyr_   r   r   r   r
   rV   Î   s    

rV   c                   @  ó<   e Zd ZdZddd„Zdd
d„Zddd„Zeddd„ƒZdS )Ú TooManySymbolOrPunctuationPlugin©Ú_punctuation_countÚ_symbol_countÚ_character_countÚ_last_printable_charÚ_frenzy_symbol_in_wordr5   r6   c                 C  s"   d| _ d| _d| _d | _d| _d S ©Nr   Fre   r8   r   r   r
   r:   ö   s
   
z)TooManySymbolOrPunctuationPlugin.__init__r$   r;   rW   r"   c                 C  sb   |  j d7  _ || jkr,|tvr,|jr|  jd7  _n|js,|jr,t|ƒs,|  jd7  _|| _dS )ú1Optimized feed using pre-computed character info.r   é   N)	rh   ri   r   r3   rf   r*   r4   r   rg   r[   r   r   r
   r\   þ   s   

z*TooManySymbolOrPunctuationPlugin.feed_infoc                 C  s   d| _ d| _d| _d S ©Nr   )rf   rh   rg   r8   r   r   r
   r]     s   
z&TooManySymbolOrPunctuationPlugin.resetr^   c                 C  s0   | j dkrdS | j| j | j  }|dkr|S dS )Nr   ç        ç333333Ó?)rh   rf   rg   )r9   Úratio_of_punctuationr   r   r
   r_     s   

þz&TooManySymbolOrPunctuationPlugin.ratioNrP   r`   ra   ©	rQ   rR   rS   rU   r:   r\   r]   rb   r_   r   r   r   r
   rd   ì   s    


rd   c                   @  rc   )ÚTooManyAccentuatedPlugin©rh   Ú_accentuated_countr5   r6   c                 C  ó   d| _ d| _d S rn   rt   r8   r   r   r
   r:   "  ó   
z!TooManyAccentuatedPlugin.__init__r$   r;   rW   r"   c                 C  s*   |  j d7  _ |jr|  jd7  _dS dS ©rl   r   N)rh   r.   ru   r[   r   r   r
   r\   &  s   ÿz"TooManyAccentuatedPlugin.feed_infoc                 C  rv   rn   rt   r8   r   r   r
   r]   -  rw   zTooManyAccentuatedPlugin.resetr^   c                 C  s*   | j dk rdS | j| j  }|dkr|S dS )Nr   ro   gffffffÖ?rt   )r9   Úratio_of_accentuationr   r   r
   r_   1  s   
zTooManyAccentuatedPlugin.ratioNrP   r`   ra   rr   r   r   r   r
   rs     s    


rs   c                   @  rc   )ÚUnprintablePlugin©Ú_unprintable_countrh   r5   r6   c                 C  rv   rn   r{   r8   r   r   r
   r:   >  rw   zUnprintablePlugin.__init__r$   r;   rW   r"   c                 C  s<   |j s|js|dkr|dkr|  jd7  _|  jd7  _dS )rl   úu   ï»¿r   N)r)   r%   r|   rh   r[   r   r   r
   r\   B  s   ÿþzUnprintablePlugin.feed_infoc                 C  s
   d| _ d S rn   )r|   r8   r   r   r
   r]   M  s   
zUnprintablePlugin.resetr^   c                 C  ó   | j dkrdS | jd | j  S )Nr   ro   r   )rh   r|   r8   r   r   r
   r_   P  ó   
zUnprintablePlugin.ratioNrP   r`   ra   rr   r   r   r   r
   rz   :  s    


rz   c                   @  rc   )ÚSuspiciousDuplicateAccentPlugin©Ú_successive_countrh   Ú_last_latin_characterÚ_last_was_accentuatedr5   r6   c                 C  ó   d| _ d| _d | _d| _d S rk   r�   r8   r   r   r
   r:   a  s   
z(SuspiciousDuplicateAccentPlugin.__init__r$   r;   rW   r"   c                 C  st   |  j d7  _ | jdur1|jr1| jr1|jr!| j ¡ r!|  jd7  _t|ƒt| jƒkr1|  jd7  _|| _|j| _dS rx   )rh   rƒ   r.   r„   r'   rI   r‚   r   r[   r   r   r
   r\   h  s   
ÿþz)SuspiciousDuplicateAccentPlugin.feed_infoc                 C  r…   rk   r�   r8   r   r   r
   r]   w  ó   
z%SuspiciousDuplicateAccentPlugin.resetr^   c                 C  r~   )Nr   ro   rm   )rh   r‚   r8   r   r   r
   r_   }  r   z%SuspiciousDuplicateAccentPlugin.ratioNrP   r`   ra   rr   r   r   r   r
   r€   X  s    


r€   c                   @  rc   )ÚSuspiciousRange©Ú"_suspicious_successive_range_countrh   Ú_last_printable_seenÚ_last_printable_ranger5   r6   c                 C  ó   d| _ d| _d | _d | _d S rn   rˆ   r8   r   r   r
   r:   Ž  r†   zSuspiciousRange.__init__r$   r;   rW   r"   c                 C  s†   |  j d7  _ |js|js|tv rd| _d| _dS | jdu r(|| _t|ƒ| _dS | j}t|ƒ}t||ƒr;|  jd7  _|| _|| _dS rx   )	rh   r)   r3   r   rŠ   r‹   r   Ú is_suspiciously_successive_ranger‰   )r9   r$   rW   Úunicode_range_aÚunicode_range_br   r   r
   r\   ”  s   



zSuspiciousRange.feed_infoc                 C  rŒ   rn   )rh   r‰   rŠ   r‹   r8   r   r   r
   r]   «  r†   zSuspiciousRange.resetr^   c                 C  s"   | j dkrdS | jd | j  }|S )NrE   ro   rm   )rh   r‰   )r9   Úratio_of_suspicious_range_usager   r   r
   r_   ±  s   
þzSuspiciousRange.ratioNrP   r`   ra   rr   r   r   r   r
   r‡   …  s    


r‡   c                   @  rc   )ÚSuperWeirdWordPlugin©Ú_word_countÚ_bad_word_countÚ_foreign_long_countÚ_is_current_word_badÚ_foreign_long_watchrh   Ú_bad_character_countÚ_buffer_lengthÚ_buffer_last_charÚ_buffer_last_char_accentuatedÚ_buffer_accent_countÚ_buffer_glyph_countÚ_buffer_upper_countr5   r6   c                 C  sR   d| _ d| _d| _d| _d| _d| _d| _d| _d | _d| _	d| _
d| _d| _d S rk   r’   r8   r   r   r
   r:   Ï  s   
zSuperWeirdWordPlugin.__init__r$   r;   rW   r"   c                 C  s  |j r@|  jd7  _|| _|jr|  jd7  _|j| _|jr%|  jd7  _| js4|j	r.|jr4|j
s4d| _|j
r>|  jd7  _dS | jsEdS |jsO|jsOt|ƒrà|  jd7  _| j}|  j|7  _|dkr–| j| dkrod| _n'| jr‡| j ¡ r‡| j|kr‡|  jd7  _d| _n| jdkr–d| _|  jd7  _|dkrµ| jrµ| jdko¨| j| dk}|sµ|  jd7  _d| _| jrÉ|  jd7  _|  j|7  _d	| _d	| _d| _d| _d	| _d| _d| _d| _dS |d
vrý|jsÿ|j�rd| _|  jd7  _|| _d	| _dS dS dS dS )rl   r   TNé   ç      à?é   r   rp   F>   Ú_ú-ú<ú=ú>ú|ú~)r&   r™   rš   r'   rž   r.   r›   rœ   r—   r/   r2   r�   r)   r3   r   r“   rh   r–   rI   r•   r”   r˜   r*   r4   )r9   r$   rW   Úbuffer_lengthÚprobable_camel_casedr   r   r
   r\   á  s†   ÿþþýÿþ


þ
ÿþ
ùzSuperWeirdWordPlugin.feed_infoc                 C  sR   d| _ d | _d| _d| _d| _d| _d| _d| _d| _d| _	d| _
d| _d| _d S rk   )r™   rš   r›   r–   r—   r”   r“   rh   r˜   r•   rœ   r�   rž   r8   r   r   r
   r]   ,  s   
zSuperWeirdWordPlugin.resetr^   c                 C  s$   | j dkr| jdkrdS | j| j S )Né
   r   ro   )r“   r•   r˜   rh   r8   r   r   r
   r_   ;  s   zSuperWeirdWordPlugin.ratioNrP   r`   ra   rr   r   r   r   r
   r‘   ½  s    


Kr‘   c                   @  s@   e Zd ZdZdZddd„Zddd„Zddd„Zeddd„ƒZ	dS )ÚCjkUncommonPluginz<
    Detect messy CJK text that probably means nothing.
    ©rh   Ú_uncommon_countr5   r6   c                 C  rv   rn   r­   r8   r   r   r
   r:   K  rw   zCjkUncommonPlugin.__init__r$   r;   rW   r"   c                 C  s,   |  j d7  _ |tvr|  jd7  _dS dS rx   )rh   r   r®   r[   r   r   r
   r\   O  s   ÿzCjkUncommonPlugin.feed_infoc                 C  rv   rn   r­   r8   r   r   r
   r]   V  rw   zCjkUncommonPlugin.resetr^   c                 C  s.   | j dk rdS | j| j  }|dkr|d S dS )Nr   ro   r    r«   r­   )r9   Úuncommon_form_usager   r   r
   r_   Z  s   
zCjkUncommonPlugin.ratioNrP   r`   ra   )
rQ   rR   rS   rT   rU   r:   r\   r]   rb   r_   r   r   r   r
   r¬   C  s    


r¬   c                   @  rc   )ÚArchaicUpperLowerPlugin©	Ú_bufÚ_character_count_since_last_sepÚ_successive_upper_lower_countÚ#_successive_upper_lower_count_finalrh   Ú_last_alpha_seenÚ_last_alpha_seen_upperÚ_last_alpha_seen_lowerÚ_current_ascii_onlyr5   r6   c                 C  s:   d| _ d| _d| _d| _d| _d | _d| _d| _d| _d S )NFr   Tr±   r8   r   r   r
   r:   t  s   
z ArchaicUpperLowerPlugin.__init__r$   r;   rW   r"   c                 C  s
  |j o|j}| }|r;| jdkr;| jdkr#|js#| js#|  j| j7  _d| _d| _d| _d| _|  j	d7  _	d| _dS | jrD|j
sDd| _| jdurj|jrO| jsU|jrg| jrg| jrc|  jd7  _d| _nd| _nd| _|  j	d7  _	|  jd7  _|| _|j| _|j| _dS )rl   r   é@   NFr   Trm   )r&   r,   r³   r*   r¹   rµ   r´   r¶   r²   rh   r+   r'   r¸   r(   r·   )r9   r$   rW   Úis_concernedÚ	chunk_sepr   r   r
   r\   ƒ  sH   
ÿþÿ
ÿÿz!ArchaicUpperLowerPlugin.feed_infoc                 C  s:   d| _ d| _d| _d| _d | _d| _d| _d| _d| _d S )Nr   FT)	rh   r³   r´   rµ   r¶   r·   r¸   r²   r¹   r8   r   r   r
   r]   °  s   
zArchaicUpperLowerPlugin.resetr^   c                 C  s   | j dkrdS | j| j  S )Nr   ro   )rh   rµ   r8   r   r   r
   r_   »  s   
zArchaicUpperLowerPlugin.ratioNrP   r`   ra   rr   r   r   r   r
   r°   f  s    


-r°   c                   @  s<   e Zd ZdZddd„Zddd„Zddd„Zeddd„ƒZdS )ÚArabicIsolatedFormPlugin©rh   Ú_isolated_form_countr5   r6   c                 C  rv   rn   r¾   r8   r   r   r
   r:   Ç  rw   z!ArabicIsolatedFormPlugin.__init__c                 C  rv   rn   r¾   r8   r   r   r
   r]   Ë  rw   zArabicIsolatedFormPlugin.resetr$   r;   rW   r"   c                 C  s.   |  j d7  _ |jt@ r|  jd7  _dS dS rx   )rh   r-   r   r¿   r[   r   r   r
   r\   Ï  s   
ÿz"ArabicIsolatedFormPlugin.feed_infor^   c                 C  s   | j dk rdS | j| j  }|S )Nr   ro   r¾   )r9   Úisolated_form_usager   r   r
   r_   Ö  s   
zArabicIsolatedFormPlugin.ratioNrP   r`   ra   )	rQ   rR   rS   rU   r:   r]   r\   rb   r_   r   r   r   r
   r½   Ã  s    


r½   é   )ÚmaxsizerŽ   ú
str | Noner�   r5   rM   c                 C  sv  | du s|du r
dS | |krdS d| v rd|v rdS d| v s"d|v r$dS d| v s,d|v r6d| v s4d|v r6dS |   d¡|  d¡}}|D ]}|tv rJqC||v rQ dS qC| dv |dv }}|s_|rid	| v sgd	|v ridS |ro|rodS d
| v swd
|v r‹d	| v sd	|v r�dS | dks‰|dkr‹dS d	| v s›d	|v s›| dv r¹|dv r¹d| v s£d|v r¥dS d| v s­d|v r¯dS | dks·|dkr¹dS dS )za
    Determine if two Unicode range seen next to each other can be considered as suspicious.
    NTFÚLatinÚ	EmoticonsÚ	Combiningú )ÚHiraganaÚKatakanaÚCJKÚHangulzBasic Latin)rÉ   rÈ   ÚPunctuationÚForms)Úsplitr   )rŽ   r�   Úkeywords_range_aÚkeywords_range_bÚelÚrange_a_jp_charsÚrange_b_jp_charsr   r   r
   r�   à  sZ   þÿÿúr�   i   çš™™™™™É?FÚdecoded_sequencer;   Úmaximum_thresholdr^   Údebugc              	   C  s’  t | ƒ}|dk rd}n	|dk rd}nd}tƒ }tƒ }tƒ }tƒ }tƒ }	tƒ }
tƒ }tƒ }t	ƒ }|j
}|j
}|j
}|j
}|	j
}|
j
}|j
}|j
}|j
}tƒ }|j}td||ƒD ]n}| ||| … D ]B}||ƒ |||ƒ |||ƒ |||ƒ |jr‚|||ƒ |||ƒ |jr¢|||ƒ |jr’|||ƒ |jrš|||ƒ |jr¢|||ƒ q`|j|j |j |j |	j |
j |j |j |j }||krÄ n/qV|dƒ |d|ƒ |d|ƒ |d|ƒ |j|j |j |j |	j |
j |j |j |j }|�rDtdƒ}| td	|› d
|› d|› �¡ |dk�r(| td| dd… › �¡ | td| dd… › �¡ |||||	|
|||f	D ]}| t|j› d|j› �¡ �q3t|dƒS )zw
    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
    iÿ  rC   rÁ   rº   r<   r   Ú
Úcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=é   zStarting with: NzEnding with: iðÿÿÿz: r   )Úlenrd   rs   rz   r€   r‡   r‘   r¬   r°   r½   r\   r"   rO   Úranger%   r&   r/   r0   r1   r_   r   Úlogr   Ú	__class__Úround)rÕ   rÖ   r×   Úseq_lenÚstepÚd_spÚd_taÚd_upÚd_sdaÚd_srÚd_swÚd_cuÚd_auÚd_aiÚ	d_sp_feedÚ	d_ta_feedÚ	d_up_feedÚ
d_sda_feedÚ	d_sr_feedÚ	d_sw_feedÚ	d_cu_feedÚ	d_au_feedÚ	d_ai_feedrW   Úinfo_updateÚblock_startr$   Úmean_mess_ratioÚloggerÚdtr   r   r
   Ú
mess_ratio*  sÒ   








€ÿþýüûúùøÿÿ


ÿþýüûúùøÿÿÿþþ

rù   )rŽ   rÃ   r�   rÃ   r5   rM   )rÔ   F)rÕ   r;   rÖ   r^   r×   rM   r5   r^   )1Ú
__future__r   ÚsysÚ	functoolsr   Úloggingr   Úversion_infoÚtypingr   Útyping_extensionsÚImportErrorÚconstantr   r   r   r   r   r   r   r   r   r   r   r   r   Úutilsr   r   r   r   r   r   r   r!   Ú__annotations__r"   rV   rd   rs   rz   r€   r‡   r‘   r¬   r°   r½   r�   rù   r   r   r   r
   Ú<module>   sT    
þ<$ 1,7 "\Iÿ