o ۭjY\@sdZddlmZddlZddlZddlZddlmZmZm Z ddl Z ddl m Z m Z ddl mZddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZmZddlmZddl m!Z!ddl"m#Z#m$Z$m%Z%m&Z&m'Z'erddl(m)Z)m*Z*m+Z+m,Z,m-Z-dEddZ.   dFdGd$d%Z/Gd&ddZ0Gd'd(d(e0Z1Gd)d*d*e0Z2eedd+  , -    dHdId9d:Z3eedd+d,ddej4ej4ddfdJdCdDZ5dS)Kz parquet compat ) annotationsN) TYPE_CHECKINGAnyLiteral)catch_warningsfilterwarnings) _get_option)lib)import_optional_dependencyAbstractMethodError)doc)find_stack_level)check_dtype_backend) DataFrame get_option) _shared_docs)arrow_table_to_pandas) IOHandles get_handle is_fsspec_urlis_urlstringify_path) DtypeBackendFilePath ReadBufferStorageOptions WriteBufferenginestrreturnBaseImplc Cs|dkrtd}|dkr>ttg}d}|D]"}z|WSty6}z|dt|7}WYd}~qd}~wwtd||dkrEtS|dkrLtStd ) zreturn our implementationautozio.parquet.enginez - NzUnable to find a usable engine; tried using: 'pyarrow', 'fastparquet'. A suitable version of pyarrow or fastparquet is required for parquet support. Trying to import the above resulted in these errors:pyarrow fastparquetz.engine must be one of 'pyarrow', 'fastparquet')r PyArrowImplFastParquetImpl ImportErrorr ValueError)rengine_classes error_msgs engine_classerrr.N/var/www/html/CropPilot/venv/lib/python3.10/site-packages/pandas/io/parquet.py get_engine4s,  r0rbFpath1FilePath | ReadBuffer[bytes] | WriteBuffer[bytes]fsrstorage_optionsStorageOptions | Nonemodeis_dirboolVtuple[FilePath | ReadBuffer[bytes] | WriteBuffer[bytes], IOHandles[bytes] | None, Any]c Cs\t|}|dur;tddd}tddd}|dur%t||jr%|r$tdn|dur1t||jjr1n tdt|j t |r}|dur}|durftd}td}z |j |\}}Wn t |j fyeYnw|dur|td}|jj|fi|pwi\}}n|rt|r|d krtd d} |s|st|trtj|st||d |d } d}| j}|| |fS) zFile handling for PyArrow.Nz pyarrow.fsignore)errorsfsspecz8storage_options not supported with a pyarrow FileSystem.z9filesystem must be a pyarrow or fsspec FileSystem, not a r$r1z8storage_options passed with buffer, or non-supported URLFis_textr5)rr isinstance FileSystemNotImplementedErrorspecAbstractFileSystemr)type__name__rfrom_uri TypeError ArrowInvalidcore url_to_fsrrosr2isdirrhandle) r2r4r5r7r8path_or_handlepa_fsr=pahandlesr.r.r/_get_path_or_handleVsf       rSc@s0eZdZed ddZd ddZddd d Zd S)r!dfrr NonecCst|ts tddS)Nz+to_parquet only supports IO with DataFrames)r@rr))rTr.r.r/validate_dataframes zBaseImpl.validate_dataframecKt|Nr )selfrTr2 compressionkwargsr.r.r/writezBaseImpl.writeNcKrWrXr )rYr2columnsr[r.r.r/readr]z BaseImpl.read)rTrr rU)rTrrX)r r)rF __module__ __qualname__ staticmethodrVr\r_r.r.r.r/r!s   c@sFeZdZdddZ     ddddZdddejddfdddZdS) r&r rUcCs&tdddddl}ddl}||_dS)Nr$z(pyarrow is required for parquet support.extrar)r pyarrow.parquet(pandas.core.arrays.arrow.extension_typesapi)rYr$pandasr.r.r/__init__s  zPyArrowImpl.__init__snappyNrTrr2FilePath | WriteBuffer[bytes]rZ str | Noneindex bool | Noner5r6partition_colslist[str] | NonecKsL||d|ddi} |dur|| d<|jjj|fi| } |jr:dt|ji} | jj } i| | } | | } t |||d|dud\}}}t |t jrjt|drjt |jttfrjt |jtrg|j}n|j}z1|dur|jjj| |f|||d|n|jjj| |f||d|W|dur|dSdS|dur|ww) Nschemapreserve_index PANDAS_ATTRSwb)r5r7r8name)rZro filesystem)rZrv)rVpoprgTable from_pandasattrsjsondumpsrqmetadatareplace_schema_metadatarSr@ioBufferedWriterhasattrrurbytesdecodeparquetwrite_to_dataset write_tableclose)rYrTr2rZrmr5rorvr[from_pandas_kwargstable df_metadataexisting_metadatamerged_metadatarOrRr.r.r/r\sj         zPyArrowImpl.writeFuse_nullable_dtypesr9 dtype_backendDtypeBackend | lib.NoDefaultcKs d|d<i} tddd} | dkrd| d<t|||dd\} } }z\|jjj| f|||d |} ttd d tt| || d }Wdn1sJwY| dkrZ|j dd d}| j j rpd| j j vrp| j j d}t ||_|W| dur{| SS| dur| ww)NTuse_pandas_metadatazmode.data_manager)silentarray split_blocksr1)r5r7)r^rvfiltersr;zmake_block is deprecated)rto_pandas_kwargsF)copys PANDAS_ATTRS)rrSrgr read_tablerrDeprecationWarningr _as_managerrqr}r{loadsrzr)rYr2r^rrrr5rvr[rmanagerrOrRpa_tableresultrr.r.r/r_sZ         zPyArrowImpl.readr rUrjNNNN)rTrr2rkrZrlrmrnr5r6rorpr rU)rr9rrr5r6r r)rFr`rarir\r no_defaultr_r.r.r.r/r&s  Er&c@s@eZdZdddZ     ddd dZ    ddddZdS)r'r rUcCstddd}||_dS)Nr%z,fastparquet is required for parquet support.rc)r rg)rYr%r.r.r/ri+s zFastParquetImpl.__init__rjNrTrrZ*Literal['snappy', 'gzip', 'brotli'] | Noner5r6c  s||d|vr|durtdd|vr|d}|dur"d|d<|dur*tdt|}t|r@tdfdd|d <nrFtd td d |jj ||f|||d |WddS1sfwYdS)N partition_onzYCannot use both partition_on and partition_cols. Use partition_cols for partitioning datahive file_scheme9filesystem is not implemented for the fastparquet engine.r=csj|dfip iS)Nrt)open)r2_r=r5r.r/Vs z'FastParquetImpl.write.. open_withz?storage_options passed with file object or non-fsspec file pathT)record)rZ write_indexr) rVr)rwrBrrr rrgr\) rYrTr2rZrmror5rvr[r.rr/r\3sB   "zFastParquetImpl.writec Ksi}|dd}|dtj} d|d<|rtd| tjur"td|dur*tdt|}d} t|rHtd} | j|d fi|pAij |d <nt |t r^t j |s^t|d d|d } | j}z|jj|fi|} | jd ||d |W| dur}| SS| dur| ww)NrFr pandas_nullszNThe 'use_nullable_dtypes' argument is not supported for the fastparquet enginezHThe 'dtype_backend' argument is not supported for the fastparquet enginerr=r1r4r>)r^rr.)rwr rr)rBrrr rr4r@rrLr2rMrrNrg ParquetFile to_pandasr) rYr2r^rr5rvr[parquet_kwargsrrrRr= parquet_filer.r.r/r_hsD     zFastParquetImpl.readrr)rTrrZrr5r6r rU)NNNN)r5r6r r)rFr`rarir\r_r.r.r.r/r'*s   8r')r5r"rjrTr$FilePath | WriteBuffer[bytes] | NonerZrlrmrnrorprv bytes | Nonec Kspt|tr|g}t|} |durtn|} | j|| f|||||d||dur6t| tjs2J| SdS)a Write a DataFrame to the parquet format. Parameters ---------- df : DataFrame path : str, path object, file-like object, or None, default None String, path object (implementing ``os.PathLike[str]``), or file-like object implementing a binary ``write()`` function. If None, the result is returned as bytes. If a string, it will be used as Root Directory path when writing a partitioned dataset. The engine fastparquet does not accept file-like objects. engine : {{'auto', 'pyarrow', 'fastparquet'}}, default 'auto' Parquet library to use. If 'auto', then the option ``io.parquet.engine`` is used. The default ``io.parquet.engine`` behavior is to try 'pyarrow', falling back to 'fastparquet' if 'pyarrow' is unavailable. When using the ``'pyarrow'`` engine and no storage options are provided and a filesystem is implemented by both ``pyarrow.fs`` and ``fsspec`` (e.g. "s3://"), then the ``pyarrow.fs`` filesystem is attempted first. Use the filesystem keyword with an instantiated fsspec filesystem if you wish to use its implementation. compression : {{'snappy', 'gzip', 'brotli', 'lz4', 'zstd', None}}, default 'snappy'. Name of the compression to use. Use ``None`` for no compression. index : bool, default None If ``True``, include the dataframe's index(es) in the file output. If ``False``, they will not be written to the file. If ``None``, similar to ``True`` the dataframe's index(es) will be saved. However, instead of being saved as values, the RangeIndex will be stored as a range in the metadata so it doesn't require much space and is faster. Other indexes will be included as columns in the file output. partition_cols : str or list, optional, default None Column names by which to partition the dataset. Columns are partitioned in the order they are given. Must be None if path is not a string. {storage_options} filesystem : fsspec or pyarrow filesystem, default None Filesystem object to use when reading the parquet file. Only implemented for ``engine="pyarrow"``. .. versionadded:: 2.1.0 kwargs Additional keyword arguments passed to the engine Returns ------- bytes if no path argument is provided else None N)rZrmror5rv)r@rr0rBytesIOr\getvalue) rTr2rrZrmr5rorvr[impl path_or_bufr.r.r/ to_parquets( A rFilePath | ReadBuffer[bytes]r^rbool | lib.NoDefaultrrr&list[tuple] | list[list[tuple]] | Nonec Ksft|} |tjurd} |dur| d7} tj| ttdnd}t|| j|f||||||d|S)a Load a parquet object from the file path, returning a DataFrame. Parameters ---------- path : str, path object or file-like object String, path object (implementing ``os.PathLike[str]``), or file-like object implementing a binary ``read()`` function. The string could be a URL. Valid URL schemes include http, ftp, s3, gs, and file. For file URLs, a host is expected. A local file could be: ``file://localhost/path/to/table.parquet``. A file URL can also be a path to a directory that contains multiple partitioned parquet files. Both pyarrow and fastparquet support paths to directories as well as file URLs. A directory path could be: ``file://localhost/path/to/tables`` or ``s3://bucket/partition_dir``. engine : {{'auto', 'pyarrow', 'fastparquet'}}, default 'auto' Parquet library to use. If 'auto', then the option ``io.parquet.engine`` is used. The default ``io.parquet.engine`` behavior is to try 'pyarrow', falling back to 'fastparquet' if 'pyarrow' is unavailable. When using the ``'pyarrow'`` engine and no storage options are provided and a filesystem is implemented by both ``pyarrow.fs`` and ``fsspec`` (e.g. "s3://"), then the ``pyarrow.fs`` filesystem is attempted first. Use the filesystem keyword with an instantiated fsspec filesystem if you wish to use its implementation. columns : list, default=None If not None, only these columns will be read from the file. {storage_options} .. versionadded:: 1.3.0 use_nullable_dtypes : bool, default False If True, use dtypes that use ``pd.NA`` as missing value indicator for the resulting DataFrame. (only applicable for the ``pyarrow`` engine) As new dtypes are added that support ``pd.NA`` in the future, the output with this option will change to use those dtypes. Note: this is an experimental option, and behaviour (e.g. additional support dtypes) may change without notice. .. deprecated:: 2.0 dtype_backend : {{'numpy_nullable', 'pyarrow'}}, default 'numpy_nullable' Back-end data type applied to the resultant :class:`DataFrame` (still experimental). Behaviour is as follows: * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame` (default). * ``"pyarrow"``: returns pyarrow-backed nullable :class:`ArrowDtype` DataFrame. .. versionadded:: 2.0 filesystem : fsspec or pyarrow filesystem, default None Filesystem object to use when reading the parquet file. Only implemented for ``engine="pyarrow"``. .. versionadded:: 2.1.0 filters : List[Tuple] or List[List[Tuple]], default None To filter out data. Filter syntax: [[(column, op, val), ...],...] where op is [==, =, >, >=, <, <=, !=, in, not in] The innermost tuples are transposed into a set of filters applied through an `AND` operation. The outer list combines these sets of filters through an `OR` operation. A single list of tuples can also be used, meaning that no `OR` operation between set of filters is to be conducted. Using this argument will NOT result in row-wise filtering of the final partitions unless ``engine="pyarrow"`` is also specified. For other engines, filtering is only performed at the partition level, that is, to prevent the loading of some row-groups and/or files. .. versionadded:: 2.1.0 **kwargs Any additional kwargs are passed to the engine. Returns ------- DataFrame See Also -------- DataFrame.to_parquet : Create a parquet object that serializes a DataFrame. Examples -------- >>> original_df = pd.DataFrame( ... {{"foo": range(5), "bar": range(5, 10)}} ... ) >>> original_df foo bar 0 0 5 1 1 6 2 2 7 3 3 8 4 4 9 >>> df_parquet_bytes = original_df.to_parquet() >>> from io import BytesIO >>> restored_df = pd.read_parquet(BytesIO(df_parquet_bytes)) >>> restored_df foo bar 0 0 5 1 1 6 2 2 7 3 3 8 4 4 9 >>> restored_df.equals(original_df) True >>> restored_bar = pd.read_parquet(BytesIO(df_parquet_bytes), columns=["bar"]) >>> restored_bar bar 0 5 1 6 2 7 3 8 4 9 >>> restored_bar.equals(original_df[['bar']]) True The function uses `kwargs` that are passed directly to the engine. In the following example, we use the `filters` argument of the pyarrow engine to filter the rows of the DataFrame. Since `pyarrow` is the default engine, we can omit the `engine` argument. Note that the `filters` argument is implemented by the `pyarrow` engine, which can benefit from multithreading and also potentially be more economical in terms of memory. >>> sel = [("foo", ">", 2)] >>> restored_part = pd.read_parquet(BytesIO(df_parquet_bytes), filters=sel) >>> restored_part foo bar 0 3 8 1 4 9 zYThe argument 'use_nullable_dtypes' is deprecated and will be removed in a future version.TzFUse dtype_backend='numpy_nullable' instead of use_nullable_dtype=True.) stacklevelF)r^rr5rrrv) r0r rwarningswarn FutureWarningrrr_) r2rr^r5rrrvrr[rmsgr.r.r/ read_parquets0 r)rrr r!)Nr1F) r2r3r4rr5r6r7rr8r9r r:)Nr"rjNNNN)rTrr2rrrrZrlrmrnr5r6rorprvrr r)r2rrrr^rpr5r6rrrrrvrrrr r)6__doc__ __future__rrr{rLtypingrrrrrrpandas._config.configr pandas._libsr pandas.compat._optionalr pandas.errorsr pandas.util._decoratorsr pandas.util._exceptionsrpandas.util._validatorsrrhrrpandas.core.shared_docsrpandas.io._utilrpandas.io.commonrrrrrpandas._typingrrrrrr0rSr!r&r'rrrr.r.r.r/s`           % ?  q X