o 6WjU@s`UddlmZddlZddlmZddlmZddlmZddl m Z m Z m Z m Z ddlmZmZmZmZmZdd lmZdd lmZmZdd lmZmZmZmZmZmZe d Z!e"Z#e#$e%d e&edddZ'de(d<         d8d9d*d+Z)         d8d:d.d/Z*         d8d;d2d3Z+         d) annotationsN) lru_cache)PathLike)BinaryIO)coherence_ratioencoding_languagesmb_encoding_languagesmerge_coherence_ratios)IANA_SUPPORTEDIANA_SUPPORTED_SIMILARTOO_BIG_SEQUENCETOO_SMALL_SEQUENCETRACE) mess_ratio) CharsetMatchCharsetMatches)any_specified_encodingcut_sequence_chunks iana_nameidentify_sig_or_bomis_multi_byte_encodingshould_strip_sig_or_bomcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)scCs t| SN)rencodingre/home/esfera/Documents/content_generation/venv/lib/python3.10/site-packages/charset_normalizer/api.py.s r)keyz list[str]IANA_SUPPORTED_MB_FIRST皙?TF皙? sequencesbytes | bytearraystepsint chunk_size thresholdfloat cp_isolationlist[str] | None cp_exclusionpreemptive_behaviourboolexplainlanguage_thresholdenable_fallbackreturnrc D Cst|ttfstdt||rtj} tt t t t |} | dkrDt d|r8tt t | tt|dddgdgS|durZtt d d |d d |D}ng}|durrtt d d |dd |D}ng}| ||krtt d||| d}| }|dkr| ||krt| |}t |tk} t |tk} | rtt d| n | rtt d| g}|rt|nd}|dur||tt d|t}g}g}t}t}i}tddt}tddt}d}t}d}d}d}d}d}d}t} t}!t|\}"}#|"dur"||"tt dt |#|"|dd|vr1|d|tD]}$|rB|$|vrBq5|rL|$|vrLq5|$|vrSq5||$d}%|"|$k}&|&odt|$}'|$dvrv|&svtt d|$q5|$dvr|&stt d|$q5|$|vrtt d|$q5|$|vrtt d|$q5zt |$}(Wnt!t"fytt d|$Yq5w|r|(stt#|$})ntt$|$})|)%|stt d|$|)|q5|r|(s||krtt d |$||q5|r|(stt d!|$q5|( o | }*zU| r/|(s/t&|'s!|dtd"n |t |#td"|$d#n3|*sb|$d$krQ|&rQt&||$d#}%|%rP|%dd%krP|%dd}%nt&|'sW|n|t |#d|$d#}%Wn+t't(fy}+zt|+t(s~tt d&|$t&|+||$WYd}+~+q5d}+~+wwt)|&sdnt |#| t| |},|(o|%duot |%| k}-|-rtt d'|$tt |,d(}.t*|.d)}.d}/d}0g}1g}2zIt+||$|,||&|'|#|(|%|* D]9}3|1|3|2||3||odt |kod)kn|2d*|kr|/d7}/|/|.ks|&r|'snqWn<t't(fyR}+z-|*r:tt d&|$t&|+||$WYd}+~+q5tt d+|$t&|+|.}/d,}0WYd}+~+nd}+~+ww|0s| r|(sz|td-dj,|$d.d/Wn#t'y}+ztt d0|$t&|+||$WYd}+~+q5d}+~+ww|2rt-|2t |2nd}4|4|ks|/|.kr:||$|$t.vr|/t.|$|%dur|(s|0t1|%|4gdftt d1|$|/t2|4d2d3d4| r8|$dd|d5d6fvr8|0s8|%durzt&|'s|n|t |#d|$d#}%Wnt't(fytt d7|$Yq5w| rd}%t||$||&g|%|d8}5|$|kr.|5}n |$dkr6|5}n|5}q5|*rvzt&|'sD|n|t |#d|$d#}%Wn%t't(fyu}+ztt d&|$t&|+||$WYd}+~+q5d}+~+ww|%durh|(sht1|%}6|3|6}7|7durh|7\}8}9}:|:r0t||$|8|&|9| r|$|ddfvr|%nd|d8};| |;||$tt d9|$t2|8d2d3d4|$|ddfvr|8d:kr|8dkrt d;|;j4|rtt t | t|;gS|!|;t |!r.|dus||vr.d|vr.d|vr.|!5}|$t&|=g}>|$dkr|=rd?|=nd}?|1D]}3||3||?}@|>|@qt6|>}A|Artt d@|A|$t||$|4|&|A| r|$|ddfvr|%nd|d8}B| |B|%dur|(s|0t1|%|4|Ad,f|r|(s|4dAkr|d7}|$|ddfvr5|4d:kr5|4dkr0t d;|Bj4|r)tt t | t|BgS|!|Bt |!rm|dusD||vrmd|vrmd|vrm|!5}lzfrom_bytes..zacp_exclusion is set. use this flag for debugging purpose. limited list of encoding excluded : %s.cSr8r9r:r;rrrr>wr?z^override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.rz>Trying to detect encoding from a tiny portion of ({}) byte(s).zIUsing lazy str decoding because the payload is quite large, ({}) byte(s).z@Detected declarative mark in sequence. Priority +1 given for %s.)maxsizezIDetected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.ascii>utf_16utf_32z\Encoding %s won't be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.>utf_7zREncoding %s won't be tested as-is because detection is unreliable without BOM/SIG.zY%s is deemed too similar to a code page that was already considered unsuited. Continuing!zESkipping %s: already fast-tracked from a similar successful encoding.z2Encoding %s does not provide an IncrementalDecoderzbSkipping %s: definitive match already found, this encoding targets different languages (%s vs %s).zXSkipping %s: already accumulated %d same-family results after definitive match (cap=%d).zCSkipping single-byte %s: multi-byte definitive match already found.gArrEuz9Code page %s does not fit given bytes sequence at ALL. %szpCode page %s is a multi byte encoding table and it appear that at least one character was encoded using n-bytes.zaLazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %sTgj@strict)errorsz^LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %szc%s was excluded because of initial chaos probing. Gave up %i time(s). Computed mean chaos is %f %%.d)ndigitsrCrDz>%s does not decode the whole payload: fallback entry withheld.)preemptive_declarationzM%s fast-tracked (identical decoded payload to a prior encoding, chaos=%f %%).r%z.Encoding detection: %s is most likely the one.zZ%s fast-skipped (identical decoded payload to a prior encoding that failed chaos probing).z=%s passed initial chaos probing. Mean measured chaos is %f %%z&{} should target any language(s) of {},z We detected language {} using {}g{Gz?css|]\}}|VqdSrr)r<_vrrr Cszfrom_bytes..)defaultg?zyDefinitive match found: %s (chaos=%.3f, coherence=%.2f). Encodings targeting different language families will be skipped.g\(\?> rEr6rCrD utf_16_be utf_16_le utf_32_be utf_32_le utf_8_sigzjMulti-byte definitive match: %s (chaos=%.3f, decoded=%d/%d=%.1f%%). Single-byte encodings will be skipped.zoEncoding detection: %s is most likely the one as we detected a BOM or SIG within the beginning of the sequence.zONothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.z7Encoding detection: %s will be used as a fallback matchz:Encoding detection: utf_8 will be used as a fallback matchz:Encoding detection: ascii will be used as a fallback matchz]Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.z=Encoding detection: Unable to determine any suitable charset.)8 isinstance bytearraybytes TypeErrorformattypeloggerlevel addHandlerexplain_handlersetLevelrlendebug removeHandlerrrlogjoinr)rr rappendsetrrrrr!addrrModuleNotFoundError ImportErrorrr intersectionstrUnicodeDecodeError LookupErrorrangemaxrdecodesumr update setdefaulthashroundgetrbestr fingerprint)Dr&r(r*r+r-r/r0r2r3r4previous_logger_levellengthis_too_small_sequenceis_too_large_sequenceprioritized_encodingsspecified_encodingtestedtested_but_hard_failuretested_but_soft_failuresoft_failure_skipsuccess_fast_trackedpayload_result_cachecached_mess_ratiocached_coherence_ratiodefinitive_match_founddefinitive_target_languages post_definitive_sb_success_countPOST_DEFINITIVE_SB_CAPmb_definitive_match_foundfallback_ascii fallback_u8fallback_specifiedresultsearly_stop_results sig_encoding sig_payload encoding_ianadecoded_payloadbom_or_sig_availablestrip_sig_or_bomis_multi_byte_decoder enc_languagesdeferred_decodinger_multi_byte_bonusmax_chunk_gave_upearly_stop_countlazy_str_hard_failure md_chunks md_ratioschunkmean_mess_ratiofallback_entry payload_hashcached cached_mess cached_cd cached_passed fast_matchprobable_resulttarget_languages cd_ratios lg_inclusionchunk_languagescd_ratios_merged current_matchbest_coherencerrr from_bytes2s                                      "                                                                 rfprc Cst|||||||||| S)z Same thing than the function from_bytes but using a file pointer that is already ready. Will not close the file pointer. )rread) rr(r*r+r-r/r0r2r3r4rrrfrom_fpsrpathstr | bytes | PathLikec CsHt|d} t| ||||||||| WdS1swYdS)z Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode. Can raise IOError. rbN)openr) rr(r*r+r-r/r0r2r3r4rrrr from_paths $rfp_or_path_or_payload!PathLike | str | BinaryIO | bytesc Cst|ttfrt|||||||||| d } | St|ttfr0t|||||||||| d } | St|||||||||| d } | S)a) Detect if the given input (file, bytes, or path) points to a binary file. aka. not a string. Based on the same main heuristic algorithms and default kwargs at the sole exception that fallbacks match are disabled to be stricter around ASCII-compatible but unlikely to be a string. ) r(r*r+r-r/r0r2r3r4)rYrorrr[rZrr) rr(r*r+r-r/r0r2r3r4guessesrrr is_binarys\- r) r"r#r$NNTFr%T)r&r'r(r)r*r)r+r,r-r.r/r.r0r1r2r1r3r,r4r1r5r)rrr(r)r*r)r+r,r-r.r/r.r0r1r2r1r3r,r4r1r5r)rrr(r)r*r)r+r,r-r.r/r.r0r1r2r1r3r,r4r1r5r) r"r#r$NNTFr%F)rrr(r)r*r)r+r,r-r.r/r.r0r1r2r1r3r,r4r1r5r1)- __future__rlogging functoolsrosrtypingrcdrrr r constantr r r rrmdrmodelsrrutilsrrrrrr getLoggerr_ StreamHandlerrb setFormatter Formattersortedr!__annotations__rrrrrrrrs           !