o 6Wj4@sUddlmZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZddlmZd d lmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"dvddZ#dwddZ$e eddxddZ%e&dde'DZ(de)d<dde(DZ*de)d<dyd d!Z+dwd"d#Z,dwd$d%Z-dwd&d'Z.dwd(d)Z/dwd*d+Z0e eddwd,d-Z1e eddwd.d/Z2e eddwd0d1Z3e eddwd2d3Z4e eddwd4d5Z5e eddwd6d7Z6e eddwd8d9Z7e eddwd:d;Z8e eddwdddQdRZ?dddVdWZ@dd[d\ZAdd]d^ZBd_ejCd`fddddeZD  fdddtduZEdS)) annotationsN) bisect_right)IncrementalDecoder)aliases) lru_cache)findall) Generator)ENCODING_MARKSIANA_SUPPORTED_SIMILARRE_POSSIBLE_ENCODING_INDICATIONUNICODE_RANGES_COMBINED_SECONDARY_RANGE_NAMESUTF8_MAXIMAL_ALLOCATIONCOMMON_CJK_CHARACTERS_LATIN_CJK_HANGUL _KATAKANA _HIRAGANA_THAI_ARABIC_ARABIC_ISOLATED_FORM_ACCENT_KEYWORDS _ACCENTUATED characterstrreturnintcCszt|}Wn tyYdSwd}d|vr|tO}d|vr$|tO}d|vr,|tO}d|vr4|tO}d|vr<|tO}d|vrD|tO}d|vrT|t O}d |vrT|t O}t D] }||vrc|t O}|SqV|S) zRCompute all name-based classification flags with a single unicodedata.name() call.rLATINCJKHANGULKATAKANAHIRAGANATHAIARABICz ISOLATED FORM) unicodedataname ValueErrorrrrrrrrrrr)rdescflagskwr,g/home/esfera/Documents/content_generation/venv/lib/python3.10/site-packages/charset_normalizer/utils.py_character_flags"s: r.boolcCtt|t@SN)r/r.rrr,r,r-is_accentuatedDr3)maxsizecCs.t|}|s |S|d}tt|ddS)N r)r& decompositionsplitchrr)r decomposedcodesr,r,r- remove_accentHs  r=ccs"|] \}}|j|j|fVqdSr1)startstop).0r' ord_ranger,r,r- Us  rBzlist[tuple[int, int, str]]_UNICODE_RANGES_SORTEDcCsg|]}|dqS)rr,)r@er,r,r- YsrEz list[int]_UNICODE_RANGE_STARTS str | NonecCs<t|}tt|d}|dkrt|\}}}||kr|SdS)zK Retrieve the Unicode range official name from a single character. r rN)ordrrFrC)r character_ordidxr>r?r'r,r,r- unicode_range\srKcCr0r1)r/r.rr2r,r,r-is_latinlr4rLcCs2t|}d|vr dSt|}|durdSd|vS)NPTF Punctuationr&categoryrKrcharacter_categorycharacter_ranger,r,r-is_punctuationps rTcCsBt|}d|vs d|vrdSt|}|durdSd|vo |dkS)NSNTFFormsLorOrQr,r,r- is_symbol~s rYcCs$t|}|dur dSd|vpd|vS)NF Emoticons Pictographs)rK)rrSr,r,r- is_emoticonsr\cCs.|s|dvr dSt|}d|vp|dvS)N>｜+<>TZ>PcPdPo)isspacer&rP)rrRr,r,r- is_separators rfcCs||kSr1)islowerisupperr2r,r,r-is_case_variablericCr0r1)r/r.rr2r,r,r-is_cjkrjrkcCr0r1)r/r.rr2r,r,r- is_hiraganarjrlcCr0r1)r/r.rr2r,r,r- is_katakanarjrmcCr0r1)r/r.rr2r,r,r- is_hangulrjrncCr0r1)r/r.rr2r,r,r-is_thairjrocCr0r1)r/r.rr2r,r,r- is_arabicrjrpcCr0r1)r/r.rr2r,r,r-is_arabic_isolated_formrjrqcCs|tvSr1)rr2r,r,r-is_cjk_uncommonsrr range_namecCs|tvSr1)r)rsr,r,r-is_unicode_range_secondaryrtcCs$| o| o|dko|dkS)Nu)re isprintabler2r,r,r-is_unprintables rx sequencebytes | bytearray search_zonec Cst|ttfs tt|}|dt||jddd}|}d|vr)d|vr)dStt |}t|dkr6dS|D]'}| dd }t D]\}}||krT|S||kr^|SqFq8dS) zW Extract using ASCII-only decoder any specified encoding in the first n-bytes. Nasciiignoreerrorscodingcharsetr-_) isinstancebytes bytearray TypeErrorlenmindecodelowerrr replaceritems) rzr|seq_len decoded_zone lowered_zoneresultsspecified_encodingencoding_alias encoding_ianar,r,r-any_specified_encodings2   rr'c CsJ|dvrdSdD]}z t||WdStttfy"YqwdS)zQ Verify is a specific encoding is a multi byte one based on it IANA name > utf_7utf_8utf_16utf_32 utf_16_be utf_16_le utf_32_be utf_32_le utf_8_sigT) _codecs_cn _codecs_hk_codecs_iso2022 _codecs_jp _codecs_kr _codecs_twF) importlib import_modulegetcodec ImportErrorAttributeError LookupError)r'providerr,r,r-is_multi_byte_encodings  rtuple[str | None, bytes]cCsJtD] }t|}t|tr|g}|D]}||r!||fSqqdS)z9 Identify and extract SIG/BOM in given sequence. )N)r rr startswith)rz iana_encodingmarksmarkr,r,r-identify_sig_or_bom/s  rrcCs|dvS)N>rrr,)rr,r,r-should_strip_sig_or_bomArurTcp_namestrictcCsN|dd}tD]\}}|||fvr|Sq |r%td|d|S)zIReturns the Python normalized encoding name (Not the IANA official name).rrzUnable to retrieve IANA for '')rrrrr()rrrrr,r,r- iana_nameEs r iana_name_a iana_name_bfloatc Cst|st|r dStd|j}td|j}|dd}|dd}d}tdD]}t|g}||||krA|d7}q,|dS)Ngz encodings.r~rrr )rrrrrangerr) rr decoder_a decoder_bid_aid_bcharacter_match_counti to_be_decodedr,r,r- cp_similarityVs    rcCs|tvo |t|vS)z Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using the function cp_similarity. )r )rrr,r,r- is_cp_similarjs rcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)slevel format_stringNonecCs:t|}||t}|t|||dSr1)logging getLoggersetLevel StreamHandler setFormatter Formatter addHandler)r'rrloggerhandlerr,r,r-set_logging_handlerus  rF sequencesroffsetsr chunk_sizebom_or_sig_availablestrip_sig_or_bom sig_payloadris_multi_byte_decoderdecoded_payloaddeferred_decodingGenerator[str, None, None]c csj|r|s|D]} || | |} | sdS| VqdS| rD|s"|n|t|d} |D]} | | | |} | s;dSt| |Vq,dS|D]l} | |}|t|dkrUqF|| | |} |re|se|| } | j||rldndd} |r| dkrt|d}|r| d||vrt| | ddD]!}|||} |r|s|| } | j|dd} | d||vrnq| VqFdS) Nr~rrrr7)rrrrr)rrrrrrrrrrrchunk base_bytes cut_sequence chunk_endchunk_partial_size_chkjr,r,r-cut_sequence_chunkssV     r)rrrr)rrrr/)rrrr)rrrrG)rsrrr/)ry)rzr{r|rrrG)r'rrr/)rzr{rr)rrrr/)T)rrrr/rr)rrrrrr)rrrrrr/)r'rrrrrrr)NF)rr{rrrrrrrr/rr/rrrr/rrGrr/rr)F __future__rrrr&bisectrcodecsrencodings.aliasesr functoolsrrertypingrconstantr r r r rrrrrrrrrrrrrr.r3r=sortedrrC__annotations__rFrKrLrTrYr\rfrirkrlrmrnrorprqrrrtrxrrrrrrrINFOrrr,r,r,r-s|      L  "                    +  )