o
    Û­j*  ã                   @   s  d Z ddlmZmZ ddlZddlZddlZddlZddl	Z	ddl
mZmZ ddlmZ e	j d¡Ze	j d¡Zdd„ Zed	d
„ ƒZee	j dddg¡e	j dg d¢¡dd„ ƒƒƒZdd„ Zdd„ Ze	j ddi eddgiƒfdddieddgiƒfdddgieddd giƒfd!dgd"d#œeddgiƒfd!dgd$d#œedejdgiƒfg¡d%d&„ ƒZd'd(„ Ze	j d)g d*¢¡d+d,„ ƒZe	j d-d"d$g¡d.d/„ ƒZd0d1„ Z e	j dg d2¢¡d3d4„ ƒZ!e	j dg d5¢¡d6d7„ ƒZ"d8d9„ Z#d:d;„ Z$e	j d<¡e	j d=d>d?g¡d@dA„ ƒƒZ%dS )BzZ
Tests encoding functionality during parsing
for all of the parsers defined in parsers.py
é    )ÚBytesIOÚTextIOWrapperN)Ú	DataFrameÚread_csvz=ignore:Passing a BlockManager to DataFrame:DeprecationWarningÚpyarrow_skipc                 C   sL   d}| }t d |¡ƒ}|j|d|d�}tddggddgd	�}t ||¡ d S )
NÚcp1255u   ×©×œ×•×�:1234
562:123ú:)ÚsepÚencodingi2  é{   u   ×©×œ×•×�Ú1234©Úcolumns©r   Úencoder   r   ÚtmÚassert_frame_equal)Úall_parsersr
   ÚparserÚdataÚresultÚexpected© r   úa/var/www/html/CropPilot/venv/lib/python3.10/site-packages/pandas/tests/io/parser/test_encoding.pyÚtest_bytes_io_input   s   r   c                 C   s@   | }t d ¡ ƒ}|j|ddd d�}tddggƒ}t ||¡ d S )Nu   Å�aski, Jan;1ú;úutf-8)r	   r
   Úheaderu   Å�aski, Jané   r   )r   r   r   r   r   r   r   r   Útest_read_csv_unicode(   s
   r   r	   ú,ú	r
   )úutf-16zutf-16lezutf-16bec              	   C   s
  | }d  d|¡}dt ¡ › d�}|ddœ}d}t |¡�_}| |¡}t|dƒ�}	|	 |¡ W d   ƒ n1 s7w   Y  tt	| |¡ƒ|d	��}
|j
|fd
|i|¤Ž}|j
|
fd
|i|¤Ž}W d   ƒ n1 shw   Y  t ||¡ W d   ƒ d S 1 s~w   Y  d S )Nz)skip this
skip this too
A,B,C
1,2,3
4,5,6r    Ú__z__.csvé   )r	   Úskiprowsr   Úwb©r
   r
   )ÚreplaceÚuuidÚuuid4r   Úensure_cleanr   ÚopenÚwriter   r   r   r   )r   r	   r
   r   r   ÚpathÚkwargsÚutf8Ú
bytes_dataÚfÚbytes_bufferr   r   r   r   r   Útest_utf16_bom_skiprows2   s(   ÿü

ÿþ"÷r4   c                 C   s6   t j |d¡}| }|j|ddd�}t|ƒdksJ ‚d S )Nzutf16_ex.txtr"   r!   )r
   r	   é2   )Úosr.   Újoinr   Úlen)r   Úcsv_dir_pathr.   r   r   r   r   r   Útest_utf16_exampleO   s   r:   c                 C   sL   t j |d¡}| }|j|d dd�}| d¡}|d d }d}||ks$J ‚d S )Núunicode_series.csvúlatin-1)r   r
   r   r   i`  u$   Ã� kÃ¶ldum klaka (Cold Fever) (1994))r6   r.   r7   r   Ú	set_index)r   r9   r.   r   r   Úgotr   r   r   r   Útest_unicode_encodingV   s   
r?   zdata,kwargs,expectedza
1Úar   z"a"
1Ú	quotecharú"zb
1ÚnamesÚbÚ1ú
1T)rC   Úskip_blank_linesFc                    sn   | }d‰ d‰‡ ‡fdd„}|j dkr"|dkr"| dd¡r"tjd	d
� |j||ƒfdˆi|¤Ž}t ||¡ d S )Nu   ï»¿r   c                    s   ˆ |    ˆ¡}t|ƒS )N)r   r   )Ú_dataÚbom_data©Úbomr0   r   r   Ú_encode_data_with_bom{   s   z,test_utf8_bom.<locals>._encode_data_with_bomÚpyarrowrF   rG   Tz,https://github.com/apache/arrow/issues/38676)Úreasonr
   )ÚengineÚgetÚpytestÚskipr   r   r   )r   r   r/   r   Úrequestr   rL   r   r   rJ   r   Útest_utf8_bomb   s   

ÿrT   c                 C   sL   t dgdgdœƒ}| }| |¡}d |¡}|jt|ƒ|d�}t ||¡ d S )Ng333333@Útest)Úmb_numÚ	multibytezmb_num,multibyte
4.8,testr'   )r   Úformatr   r   r   r   r   )r   Ú	utf_valueÚencoding_fmtr   r   r
   r   r   r   r   r   Útest_read_csv_utf_aliases‹   s   

r[   zfile_path,encoding)))Úior   Úcsvz	test1.csvr   ))r\   r   r   r;   r<   ))r\   r   r   zsauron.SHIFT_JIS.csvÚshiftjisc           
      C   s  | }||Ž }|j ||d�}t||d��}|  |¡}|jrJ ‚W d   ƒ n1 s(w   Y  t ||¡ t|dd��}	|j |	|d�}|	jrFJ ‚W d   ƒ n1 sPw   Y  t ||¡ t|ddd��}	|j |	|d�}|	jroJ ‚W d   ƒ n1 syw   Y  t ||¡ d S )Nr'   Úrb)Úmoder   )r`   Ú	buffering)r   r,   Úclosedr   r   )
r   Ú	file_pathr
   Údatapathr   Úfpathr   Úfar   Úfbr   r   r   Útest_binary_mode_file_buffers—   s$   
þþþrh   Úpass_encodingc           	      C   s¬   | }|  |¡}|jdkr|du r|dv rt d¡ tddgiƒ}tjd|dd��$}| d	¡ | d
¡ |j	||r:|nd d�}t 
||¡ W d   ƒ d S 1 sOw   Y  d S )NrM   T)é   é    zThese cases freezeÚfooÚbarzw+)r`   r
   Úreturn_filelikezfoo
barr   r'   )rX   rO   rQ   rR   r   r   r+   r-   Úseekr   r   )	r   rY   rZ   ri   r   r
   r   r2   r   r   r   r   Útest_encoding_temp_file·   s   



"ûrp   c                 C   s”   | }d}d}d}t ||giƒ}t ¡ �-}| |› d|› � |¡¡ | d¡ |j||d�}t ||¡ |j	r8J ‚W d   ƒ d S 1 sCw   Y  d S )Nz	shift-jisu	   ã�¦ã�™ã�¨u   ã�“ã‚€Ú
r   r'   )
r   ÚtempfileÚNamedTemporaryFiler-   r   ro   r   r   r   rb   )r   r   r
   Útitler   r   r2   r   r   r   r   Útest_encoding_named_temp_fileË   s   

"ùru   )r   r"   z	utf-16-bez	utf-16-lezutf-32c                 C   sR   d}t | | ¡ƒ}t|d| d�}tddgddgdd	ggd
dgd�}t ||¡ d S )Nu   a	b
ï¼šfoo	0
bar	1
baz	2r!   )Ú	delimiterr
   u   ï¼šfoor   rm   r   Úbazr$   r@   rD   )r   r   r   )r
   r   Úencoded_datar   r   r   r   r   Ú%test_parse_encoded_special_charactersß   s   þry   )r   Nr"   r   r<   c              	   C   sÖ   | }t g d¢g d¢g d¢dœƒ}t ¡ �G}|j|d|d� |jdkrLd}tjt|d	�� |j||d
d� W d   ƒ n1 s>w   Y  	 W d   ƒ d S |j||d
d�}W d   ƒ n1 s^w   Y  t 	||¡ d S )N)ÚRaphaelÚ	DonatellozMiguel AngelÚLeonardo)ÚredÚpurpleÚorangeÚblue)Úsaizbo staffÚnunchunkÚkatana)ÚnameÚmaskÚweaponF)Úindexr
   rM   úBThe 'memory_map' option is not supported with the 'pyarrow' engine©ÚmatchT)r
   Ú
memory_map)
r   r   r+   Úto_csvrO   rQ   ÚraisesÚ
ValueErrorr   r   )r   r
   r   r   ÚfileÚmsgÚdfr   r   r   Útest_encoding_memory_mapð   s&   ýÿ

ÿù	÷
r’   c              	   C   sØ   | }t dgd d�}d|jd< t d¡�H}|j|dddd	� |jd
krMd}tjt|d�� |j	|ddd� W d  ƒ n1 s?w   Y  	 W d  ƒ dS |j	|ddd�}W d  ƒ n1 s_w   Y  t 
||¡ dS )zO
    Chunk splits a multibyte character with memory_map=True

    GH 43540
    Úaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaai   )r   u�   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaÄ…iÿ  zbug-gh43540.csvFr   ©r‡   r   r
   rM   rˆ   r‰   NT)r   r‹   )r   Úilocr   r+   rŒ   rO   rQ   r�   rŽ   r   r   )r   r   r‘   Úfnamer�   Údfrr   r   r   Ú test_chunk_splits_multibyte_char  s   

ÿù	÷
r˜   c              	   C   sB  g }d}d}d}t t|ƒt|ƒ|ƒD ])}d dd„ t ||d ƒD ƒ¡d }z| d¡ W n	 ty5   Y qw | |¡ q| }t|ƒ}t d	¡�J}	|j	|	d
d
dd� |j
dkr�d}
tjt|
d�� |j|	dddd� W d  ƒ n1 ssw   Y  	 W d  ƒ dS |j|	dddd�}W d  ƒ n1 s”w   Y  t ||¡ dS )zg
    GH 43787

    Test correct handling of UTF-8 chars when memory_map=True and encoding is UTF-8
    é€   ú u   ð�‚€Ú c                 S   s   g | ]}t |ƒ‘qS r   )Úchr)Ú.0Úcr   r   r   Ú
<listcomp>/  s    z,test_readcsv_memmap_utf8.<locals>.<listcomp>rq   r   zutf8test.csvFr”   rM   rˆ   r‰   NT)r   r‹   r
   )ÚrangeÚordr7   r   ÚUnicodeEncodeErrorÚappendr   r   r+   rŒ   rO   rQ   r�   rŽ   r   r   )r   ÚlinesÚline_lengthÚ
start_charÚend_charÚlnumÚliner   r‘   r–   r�   r—   r   r   r   Útest_readcsv_memmap_utf8"  s4   "ÿ
ÿù	÷
rª   Úpyarrow_xfailr`   zw+bzw+tc                 C   s~   | }d}d|v r
d}t j|dd��}| |¡ | d¡ | |¡}W d   ƒ n1 s+w   Y  tg dgd�}t ||¡ d S )Ns   abcdÚtÚabcdr   )r`   r
   r   r   )rr   ÚSpooledTemporaryFiler-   ro   r   r   r   r   )r   r`   r   ÚcontentÚhandler‘   r   r   r   r   Útest_not_readableD  s   

ýr±   )&Ú__doc__r\   r   r   r6   rr   r)   ÚnumpyÚnprQ   Úpandasr   r   Úpandas._testingÚ_testingr   ÚmarkÚfilterwarningsÚ
pytestmarkÚusefixturesÚskip_pyarrowr   r   Úparametrizer4   r:   r?   ÚnanrT   r[   rh   rp   ru   ry   r’   r˜   rª   r±   r   r   r   r   Ú<module>   sn    ÿ
	
ýöþ
þ

ÿ


"