o
    Û­j;  ã                	   @  s8  d Z ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl	Z	ddl
ZddlmZ ddlmZ ddlZdd	lmZ dd
lmZ erPddlmZmZmZmZ dZdZdZdZg d¢ZdZ dZ!dZ"dZ#de › de"› de!› de#› d�	Z$de › de!› d�Z%dZ&d*dd „Z'd+d"d#„Z(d$d%„ Z)d&d'„ Z*G d(d)„ d)eej+ƒZ,dS ),a-  
Read a SAS XPort format file into a Pandas DataFrame.

Based on code from Jack Cushman (github.com/jcushman/xport).

The file format is defined here:

https://support.sas.com/content/dam/SAS/support/en/technical-papers/record-layout-of-a-sas-version-5-or-6-data-set-in-sas-transport-xport-format.pdf
é    )Úannotations)Úabc)ÚdatetimeN)ÚTYPE_CHECKING)ÚAppender)Úfind_stack_level)Ú
get_handle)Ú
ReaderBase)ÚCompressionOptionsÚDatetimeNaTTypeÚFilePathÚ
ReadBufferzPHEADER RECORD*******LIBRARY HEADER RECORD!!!!!!!000000000000000000000000000000  zKHEADER RECORD*******MEMBER  HEADER RECORD!!!!!!!000000000000000001600000000zPHEADER RECORD*******DSCRPTR HEADER RECORD!!!!!!!000000000000000000000000000000  zPHEADER RECORD*******OBS     HEADER RECORD!!!!!!!000000000000000000000000000000  )ÚntypeÚnhfunÚfield_lengthÚnvar0ÚnameÚlabelÚnformÚnflÚnum_decimalsÚnfjÚnfillÚniformÚniflÚnifdÚnposÚ_z‚Parameters
----------
filepath_or_buffer : str or file-like object
    Path to SAS file or object implementing binary read method.zâindex : identifier of index column
    Identifier of column that should be used as index of the DataFrame.
encoding : str
    Encoding for text data.
chunksize : int
    Read file `chunksize` lines at a time, returns iterator.zBformat : str
    File format, only `xport` is currently supported.z\iterator : bool, default False
    Return XportReader object for reading file incrementally.z#Read a SAS file into a DataFrame.

Ú
a  

Returns
-------
DataFrame or XportReader

Examples
--------
Read a SAS Xport file:

>>> df = pd.read_sas('filename.XPT')

Read a Xport file in 10,000 line chunks:

>>> itr = pd.read_sas('filename.XPT', chunksize=10000)
>>> for chunk in itr:
>>>     do_something(chunk)

z$Class for reading SAS Xport files.

zš

Attributes
----------
member_info : list
    Contains information about the file
fields : list
    Contains information about the variables in the file
zÊRead observations from SAS Xport file, returning as data frame.

Parameters
----------
nrows : int
    Number of rows to read from data file; if None, read whole
    file.

Returns
-------
A DataFrame.
ÚdatestrÚstrÚreturnr   c                 C  s(   zt  | d¡W S  ty   tj Y S w )z1Given a date in xport format, return Python date.z%d%b%y:%H:%M:%S)r   ÚstrptimeÚ
ValueErrorÚpdÚNaT)r   © r&   úT/var/www/html/CropPilot/venv/lib/python3.10/site-packages/pandas/io/sas/sas_xport.pyÚ_parse_date“   s
   
ÿr(   Úsc                 C  s@   i }d}|D ]\}}| ||| …   ¡ ||< ||7 }q|d= |S )a  
    Parameters
    ----------
    s: str
        Fixed-length string to split
    parts: list of (name, length) pairs
        Used to break up string, name '_' will be filtered from output.

    Returns
    -------
    Dict of name:contents of string at given location.
    r   r   )Ústrip)r)   ÚpartsÚoutÚstartr   Úlengthr&   r&   r'   Ú_split_lineœ   s   
r/   c                 C  sT   |dkr(t  t| ƒt  d¡¡}t  d|› dd| › �¡}|j|d�}| |d< |S | S )Né   ÚS8ÚSz,S©ÚdtypeÚf0)ÚnpÚzerosÚlenr4   Úview)ÚvecÚnbytesÚvec1r4   Úvec2r&   r&   r'   Ú_handle_truncated_float_vec²   s   	r>   c           	      C  s  t  d¡}| j|d�}|d }|d }|d@ }t jt| ƒt jd�}d|t  |d@ ¡< d|t  |d	@ ¡< d
|t  |d@ ¡< ||L }||? |d@ dd
|  > B }|dM }||d? d@ d d> | d d> |d@ B O }t jt|ƒfdd�}||d< ||d< |jdd�}| d¡}|S )zf
    Parse a vector of float values representing IBM 8 byte floats into
    native 8 byte floats.
    z>u4,>u4r3   r5   Úf1iÿÿÿ é   i    é   i  @ é   i  € é   é   l   ÿß é   é   éA   iÿ  é   l        z>f8Úf8)	r6   r4   r9   r7   r8   Úuint8ÚwhereÚemptyÚastype)	r:   r4   r<   Úxport1Úxport2Úieee1ÚshiftÚieee2Úieeer&   r&   r'   Ú_parse_float_vecÅ   s*   
	 	ÿ
rT   c                   @  s|   e Zd ZeZ				d$d%dd„Zd&dd„Zdd„ Zd&dd„Zd'dd„Z	d(dd„Z
d)d*dd„Zdd „ Zeeƒd)d+d"d#„ƒZdS ),ÚXportReaderNú
ISO-8859-1ÚinferÚfilepath_or_bufferúFilePath | ReadBuffer[bytes]Úencodingú
str | NoneÚ	chunksizeú
int | NoneÚcompressionr
   r!   ÚNonec                 C  s^   || _ d| _|| _|| _t|d|d|d�| _| jj| _z|  ¡  W d S  t	y.   |  
¡  ‚ w )Nr   ÚrbF)rZ   Úis_textr^   )Ú	_encodingÚ_lines_readÚ_indexÚ
_chunksizer   ÚhandlesÚhandlerX   Ú_read_headerÚ	ExceptionÚclose)ÚselfrX   ÚindexrZ   r\   r^   r&   r&   r'   Ú__init__  s$   û
þzXportReader.__init__c                 C  s   | j  ¡  d S ©N)rf   rj   ©rk   r&   r&   r'   rj     s   zXportReader.closec                 C  s   | j  d¡ ¡ S )NéP   )rX   ÚreadÚdecodero   r&   r&   r'   Ú_get_row   s   zXportReader._get_rowc              	   C  sF  | j  d¡ |  ¡ }|tkrd|v rtdƒ‚tdƒ‚|  ¡ }ddgddgd	dgd
dgddgg}t||ƒ}|d dkr>tdƒ‚t|d ƒ|d< || _|  ¡ }t|d d… ƒ|d< |  ¡ }|  ¡ }| t	¡}|t
k}	|rl|	sptdƒ‚t|dd… ƒ}
ddgddgddgddgd	dgd
dgddgg}t|  ¡ |ƒ}ddgd
dgddgddgg}| t|  ¡ |ƒ¡ t|d ƒ|d< t|d ƒ|d< || _dddœ}t|  ¡ dd… ƒ}|
| }|d rà|d|d  7 }| j  |¡}g }d}t|ƒ|
k�rh|d |
… ||
d … }}| d¡}t d|¡}ttt|ƒƒ}|d
= ||d   |d < |d! }|d  dk�r:|d"k �s0|dk�r:d#|› d$�}t|ƒ‚| ¡ D ]\}}z	| ¡ ||< W �q> t�yV   Y �q>w ||d! 7 }||g7 }t|ƒ|
ksñ|  ¡ }|tk�sutd%ƒ‚|| _|| _| j  ¡ | _|  ¡ | _ d&d'„ | jD ƒ| _!d(d'„ t"| jƒD ƒ}t# $|¡}|| _%d S ))Nr   z**COMPRESSED**z<Header record indicates a CPORT file, which is not readable.z#Header record is not an XPORT file.ÚprefixrE   Úversionr0   ÚOSr   Úcreatedé   zSAS     SAS     SASLIBz!Header record has invalid prefix.ÚmodifiedzMember header not foundéûÿÿÿéþÿÿÿÚset_nameÚsasdatar   é(   ÚtypeÚnumericÚchar)r@   rA   é6   é:   rp   éŒ   z>hhhh8s40s8shhh2s8shhl52sr   r   rA   zFloating field width z is not between 2 and 8.zObservation header not found.c                 S  s   g | ]}|d    ¡ ‘qS )r   )rr   )Ú.0Úxr&   r&   r'   Ú
<listcomp>‡  ó    z,XportReader._read_header.<locals>.<listcomp>c                 S  s,   g | ]\}}d t |ƒ dt |d ƒ f‘qS )r)   r2   r   )r    )r…   ÚiÚfieldr&   r&   r'   r‡   Š  s    ÿÿ)&rX   Úseekrs   Ú_correct_line1r#   r/   r(   Ú	file_infoÚ
startswithÚ_correct_header1Ú_correct_header2ÚintÚupdateÚmember_inforq   r8   ÚljustÚstructÚunpackÚdictÚzipÚ
_fieldkeysÚ	TypeErrorÚitemsr*   ÚAttributeErrorÚ_correct_obs_headerÚfieldsÚrecord_lengthÚtellÚrecord_startÚ_record_countÚnobsÚcolumnsÚ	enumerater6   r4   Ú_dtype)rk   Úline1Úline2Úfifr�   Úline3Úheader1Úheader2Ú	headflag1Ú	headflag2ÚfieldnamelengthÚmemr“   ÚtypesÚ
fieldcountÚ
datalengthÚ	fielddatarž   Ú
obs_lengthÚ
fieldbytesÚfieldstructrŠ   ÚflÚmsgÚkÚvÚheaderÚdtypelr4   r&   r&   r'   rh   #  s¢   ÿ"

ù	


þ
"ÿ
å

þ

zXportReader._read_headerúpd.DataFramec                 C  s   | j | jpdd�S )Nr@   ©Únrows)rq   re   ro   r&   r&   r'   Ú__next__‘  s   zXportReader.__next__r‘   c                 C  sÎ   | j  dd¡ | j  ¡ | j }|d dkrtjdtƒ d� | jdkr.| j  | j¡ || j S | j  dd¡ | j  d¡}t	j
|t	jd�}t	 |dk¡}t|ƒdkrSd}nd	t|ƒ }| j  | j¡ || | j S )
zÇ
        Get number of records in file.

        This is maybe suboptimal because we have to seek to the end of
        the file.

        Side effect: returns file position to record_start.
        r   rA   rp   zxport file may be corrupted.)Ú
stackleveli°ÿÿÿr3   l     @@€  r0   )rX   r‹   r    r¡   ÚwarningsÚwarnr   rŸ   rq   r6   Ú
frombufferÚuint64Úflatnonzeror8   )rk   Útotal_records_lengthÚlast_card_bytesÚ	last_cardÚixÚtail_padr&   r&   r'   r¢   ”  s&   	þ

zXportReader._record_countÚsizec                 C  s   |du r| j }| j|d�S )a  
        Reads lines from Xport file and returns as dataframe

        Parameters
        ----------
        size : int, defaults to None
            Number of lines to read.  If None, reads whole file.

        Returns
        -------
        DataFrame
        Nr¿   )re   rq   )rk   rÍ   r&   r&   r'   Ú	get_chunkº  s   zXportReader.get_chunkc                 C  sl   |j dd�}|d dk|d dk@ |d dk@ }|d dk|d d	k@ |d d
kB |d dkB }||M }|S )Nzu1,u1,u2,u4r3   r?   r   Úf2Úf3r5   rG   éZ   é_   é.   )r9   )rk   r:   r»   ÚmissÚmiss1r&   r&   r'   Ú_missing_doubleË  s   $
ÿ
þÿzXportReader._missing_doublerÀ   c                   sd  |d u rˆ j }t|ˆ j ˆ j ƒ}|ˆ j }|dkrˆ  ¡  t‚ˆ j |¡}tj	|ˆ j
|d�}i }tˆ jƒD ]U\}}|dt|ƒ  }	ˆ j| d }
|
dkret|	ˆ j| d ƒ}	ˆ  |	¡}t|	ƒ}tj||< nˆ j| d dkrƒdd	„ |	D ƒ}ˆ jd urƒ‡ fd
d	„|D ƒ}| ||i¡ q5t |¡}ˆ jd u r£t tˆ jˆ j| ƒ¡|_n| ˆ j¡}ˆ  j|7  _|S )Nr   )r4   Úcountr)   r   r€   r   r�   c                 S  s   g | ]}|  ¡ ‘qS r&   )Úrstrip©r…   Úyr&   r&   r'   r‡   í  s    z$XportReader.read.<locals>.<listcomp>c                   s   g | ]}|  ˆ j¡‘qS r&   )rr   rb   rÙ   ro   r&   r'   r‡   ð  rˆ   )r£   Úminrc   rŸ   rj   ÚStopIterationrX   rq   r6   rÅ   r¦   r¥   r¤   r    rž   r>   rÖ   rT   Únanrb   r’   r$   Ú	DataFramerd   ÚIndexÚrangerl   Ú	set_index)rk   rÀ   Ú
read_linesÚread_lenÚrawÚdataÚdf_dataÚjr†   r:   r   rÔ   r»   Údfr&   ro   r'   rq   Ö  s:   




zXportReader.read)NrV   NrW   )
rX   rY   rZ   r[   r\   r]   r^   r
   r!   r_   )r!   r_   )r!   r¾   )r!   r‘   rn   )rÍ   r]   r!   r¾   )rÀ   r]   r!   r¾   )Ú__name__Ú
__module__Ú__qualname__Ú_xport_reader_docÚ__doc__rm   rj   rs   rh   rÁ   r¢   rÎ   rÖ   r   Ú_read_method_docrq   r&   r&   r&   r'   rU   þ   s     ú


n
&rU   )r   r    r!   r   )r)   r    )-rí   Ú
__future__r   Úcollectionsr   r   r•   Útypingr   rÃ   Únumpyr6   Úpandas.util._decoratorsr   Úpandas.util._exceptionsr   Úpandasr$   Úpandas.io.commonr   Úpandas.io.sas.sasreaderr	   Úpandas._typingr
   r   r   r   rŒ   r�   r�   r�   r™   Ú_base_params_docÚ_params2_docÚ_format_params_docÚ_iterator_docÚ_read_sas_docrì   rî   r(   r/   r>   rT   ÚIteratorrU   r&   r&   r&   r'   Ú<module>   s`    	ÿÿÿÿþýüûýü

	9