uj;oDdZddlmZddlZddlZddlZddlmZmZm Z ddl Z ddl m Z m Z ddl mZddlmZddlmZdd lmZmZdd lmZdd lmZdd lmZdd lmZddlm Z m!Z!ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(erddl)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/ddZ0 d ddZ1GddZ2Gdde2Z3Gdde2Z4dejjdddddf d dZ6edejjddejjdddf d!dZ7y)"zparquet compat) annotationsN) TYPE_CHECKINGAnyLiteral)catch_warningsfilterwarnings)_global_config)lib)import_optional_dependency)AbstractMethodErrorPandas4Warning) set_module)find_stack_level)check_dtype_backend) DataFrame)arrow_table_to_pandassuppress_pyarrow_values_warning) IOHandlescheck_parent_directory get_handle is_fsspec_urlis_urlstringify_path) DtypeBackendFilePathParquetCompressionOptions ReadBufferStorageOptions WriteBufferBaseImplct|dk(rtddd}|dk(r,ttg}d}|D] } |cStd||d k(r tS|d k(r.t j d tt tStd #t$r}|dt |zz }Yd}~d}~wwxYw)zreturn our implementationautoioparquetenginez - NzUnable to find a usable engine; tried using: 'pyarrow', 'fastparquet'. A suitable version of pyarrow or fastparquet is required for parquet support. Trying to import the above resulted in these errors:pyarrow fastparquetzfThe 'fastparquet' engine is deprecated and will be removed in a future version. Use 'pyarrow' instead. stacklevelz.engine must be one of 'pyarrow', 'fastparquet') config PyArrowImplFastParquetImpl ImportErrorstrwarningswarnr r ValueError)r%engine_classes error_msgs engine_classerrs \/var/www/html/python-recuperauto-other/env/lib/python3.12/site-packages/pandas/io/parquet.py get_enginer89s i(2 %7 * 1L 1#~% 1  C l    } =   5 ')    E FF1 1gC00  1sB B7B22B7ct|}|tdd}tdd}|#t||jr |rOt d|!t||j j rn!tdt|jt|rn|ld}|5td} td} |jj|\}}|N td}|j"j$|fi|xsi\}}n|rt)|r|d k7r td d} |s|st|trt*j,j/|st)|rt1||d | } | j2}nft*j,j5|}d|vsd|vsd|vr;t7|td} | j9||} t;| ddi| g} | }|| |fS#t| jf$rY/t$r(} t|j!d s| }Yd} ~ Zd} ~ wwxYw#t&$r } || |d} ~ wwxYw)zFile handling for PyArrow.Nz pyarrow.fsignore)errorsfsspecz8storage_options not supported with a pyarrow FileSystem.z9filesystem must be a pyarrow or fsspec FileSystem, not a r'zhdfs:///rbz8storage_options passed with buffer, or non-supported URLFis_textstorage_optionswaxmethod)handle compressioncreated_handles)rr isinstance FileSystemNotImplementedErrorspecAbstractFileSystemr2type__name__rfrom_uri TypeErrorArrowExceptionOSErrorr/ startswithcore url_to_fs ExceptionrospathisdirrrE expanduserrOSFiler) rXfsr@modeis_dirpath_or_handlepa_fsr<pa_errorpar6handlesstreams r7_get_path_or_handlereasv$D)N ~*<I+HXF  B0@0@!A)N Jr6;;3Q3Q$R b**+- ^$  "+I6B.|%>t%D"N : (3H=%:V[[%:%:"&'6'<"&"N &"8DDLSTTG  ~s + n- . !!e_G%^^N WW//?Nd{cTkSD[&~6 0 :>48#!!)4 0%+H "( 7B &&yr001  >*55jA   (#x'  (s69G./H9.H6H6H11H69 I II c0eZdZeddZddZdddZy)r c:t|ts tdy)Nz+to_parquet only supports IO with DataFrames)rHrr2)dfs r7validate_dataframezBaseImpl.validate_dataframes"i(JK K)c t|Nr )selfrhrXrFkwargss r7writezBaseImpl.write !$''rjNc t|rlrm)rnrXcolumnsros r7readz BaseImpl.readrqrj)rhrreturnNonerl)rur)rN __module__ __qualname__ staticmethodrirprtrjr7r r s LL((rjczeZdZddZ d ddZddej dddf ddZy) r,c<tddddl}ddl}||_y)Nr'z(pyarrow is required for parquet support.extrar)r pyarrow.parquet(pandas.core.arrays.arrow.extension_typesapi)rnr'pandass r7__init__zPyArrowImpl.__init__s!" G   8rjNc |j|d|jddi} ||| d<t5|jjj |fi| } ddd|j rNdtj|j i}  jj} i| | } | j| } t|||d|du\}}} |-|jjj |f|||d|n+|jjj |f||d|||j!yy#1swYxYw#||j!wwxYw)Nschemapreserve_index PANDAS_ATTRSwb)r@r]r^)rFpartition_cols filesystem)rFr)ripoprrTable from_pandasattrsjsondumpsrmetadatareplace_schema_metadatarer$write_to_dataset write_tableclose)rnrhrXrFindexr@rrrofrom_pandas_kwargstable df_metadataexisting_metadatamerged_metadatar_rcs r7rpzPyArrowImpl.writes #.6 8T8R-S  38 / 0 , . I.DHHNN..rH5GHE I 88)4::bhh+?@K % 5 5 B!2BkBO11/BE.A  +!- / + )1  11"!,#1)   -  ,,"!,)   " #K I IJ" #s(EAEEE'c *d|d<t|||d\} } } |jjj| f|||d|} d} | jj r1d| jj vr| jj d} t 5tddtt| || } ddd| tj|  _  | | jSS#1swY:xYw#| | jwwxYw) NTuse_pandas_metadatar=)r@r])rsrfilterss PANDAS_ATTRSr:make_block is deprecated) dtype_backendto_pandas_kwargs)rerr$ read_tablerrrrr rrloadsrr)rnrXrsrrr@rrror_rcpa_tablerresults r7rtzPyArrowImpl.reads))-$%.A  + / +  2txx''22%   HK''"hoo&>&>>"*//":":?"KK! ." /"/%5 &#zz+6 " #!   " #s$A>C= C18%C=1C:6C==DrurvsnappyNNNN)rhrrXzFilePath | WriteBuffer[bytes]rFrr bool | Noner@StorageOptions | Nonerlist[str] | Nonerurv)rDtypeBackend | lib.NoDefaultr@rrdict[str, Any] | Nonerur)rNrwrxrrpr no_defaultrtrzrjr7r,r,s 2:!15+/7 7 ,7 / 7  7 / 7 )7  7 x69nn15262 4 2 / 2 02  2 rjr,cTeZdZddZ d ddZ d d dZy) r-c,tdd}||_y)Nr(z,fastparquet is required for parquet support.r})r r)rnr(s r7rzFastParquetImpl.__init__Js1 !O rjNc  |j|d|vr | tdd|vr|jd}|d|d<| tdt |}t |rt d fd|d<n r td td 5|jj||f|||d |dddy#1swYyxYw) N partition_onzYCannot use both partition_on and partition_cols. Use partition_cols for partitioning datahive file_scheme9filesystem is not implemented for the fastparquet engine.r<cPj|dfixsijS)Nr)open)rX_r<r@s r7z'FastParquetImpl.write..us.+&++d3.4"3dfrj open_withz?storage_options passed with file object or non-fsspec file pathT)record)rF write_indexr) rir2rrJrrr rrrp) rnrhrXrFrrr@rror<s ` @r7rpzFastParquetImpl.writeRs # V #(BK  V ##ZZ7N  %$*F= !  !%K  d#  /9F#F; Q 4 (  DHHNN (!+        s #B>>Cc ,i}|jdtj} d|d<| tjur td| t d| t dt |}d} t |r1td} | j|dfi|xsij|d <nJt|tr:tjj|st|dd| } | j } |j"j$|fi|} t'5t)d d t*| j,d||d |cddd| | j/SS#1swYnxYw | | j/yy#| | j/wwxYw)NrF pandas_nullszHThe 'dtype_backend' argument is not supported for the fastparquet enginerz?to_pandas_kwargs is not implemented for the fastparquet engine.r<r=r\r>r:r)rsrrz)rr rr2rJrrr rr\rHr/rWrXrYrrEr ParquetFilerrr to_pandasr) rnrXrsrr@rrroparquet_kwargsrrcr< parquet_files r7rtzFastParquetImpl.reads*, ?CNNC ).~&  .%   !%K   '%Q d#  /9F#.6;;tT#Uo>SQS#U#X#XN4 c "277==+>!dE?G>>D /488//GGL! ." .|--#W8>   " #   " #w" #s$3'E>&E E>E&"E>>Frr)rhrrFz*Literal['snappy', 'gzip', 'brotli'] | Noner@rrurv)NNNNN)r@rrrrur)rNrwrxrrprtrzrjr7r-r-Is{CK153 3@ 3/3 3p15267 / 7 07  7 rjr-rc t|tr|g}|tjur)|dk(r$t j dt t|tjurd}t|} |tjn|} | j|| f|||||d||,t| tjsJ| jSy)a Write a DataFrame to the parquet format. Parameters ---------- df : DataFrame path : str, path object, file-like object, or None, default None String, path object (implementing ``os.PathLike[str]``), or file-like object implementing a binary ``write()`` function. If None, the result is returned as bytes. If a string, it will be used as the root directory path when writing a partitioned dataset. The engine fastparquet does not accept file-like objects. The string could be a URL. Valid URL schemes include http, ftp, s3, gs, and file. For file URLs, a host is expected. A local file could be: ``file://localhost/path/to/table.parquet``. A remote example could be: ``s3://bucket/path/to/table.parquet``. Certain URL schemes may require additional packages. For example, S3 URLs require the ``s3fs`` library. See :ref:`install.optional_dependencies` for a full list. engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto' Parquet library to use. If 'auto', then the option ``io.parquet.engine`` is used. The default ``io.parquet.engine`` behavior is to try 'pyarrow', falling back to 'fastparquet' if 'pyarrow' is unavailable. When using the ``'pyarrow'`` engine and no storage options are provided and a filesystem is implemented by both ``pyarrow.fs`` and ``fsspec`` (e.g. "s3://"), then the ``pyarrow.fs`` filesystem is attempted first. Use the filesystem keyword with an instantiated fsspec filesystem if you wish to use its implementation. .. deprecated:: 3.1.0 The ``'fastparquet'`` and ``'auto'`` engine options are deprecated. Use ``'pyarrow'`` or do not pass ``engine`` to use the default. compression : {'snappy', 'gzip', 'brotli', 'lz4', 'zstd', None}, default 'snappy'. Name of the compression to use. Use ``None`` for no compression. index : bool, default None If ``True``, include the dataframe's index(es) in the file output. If ``False``, they will not be written to the file. If ``None``, similar to ``True`` the dataframe's index(es) will be saved. However, instead of being saved as values, the RangeIndex will be stored as a range in the metadata so it doesn't require much space and is faster. Other indexes will be included as columns in the file output. partition_cols : str or list, optional, default None Column names by which to partition the dataset. Columns are partitioned in the order they are given. Must be None if path is not a string. storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc. For HTTP(S) URLs the key-value pairs are forwarded to ``urllib.request.Request`` as header options. For other URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more details, and for more examples on storage options refer `here `_. filesystem : fsspec or pyarrow filesystem, default None Filesystem object to use when reading the parquet file. Only implemented for ``engine="pyarrow"``. .. versionadded:: 2.1.0 **kwargs Additional keyword arguments passed to the engine: * For ``engine="pyarrow"``: passed to :func:`pyarrow.parquet.write_table` or :func:`pyarrow.parquet.write_to_dataset` (when using partition_cols) * For ``engine="fastparquet"``: passed to :func:`fastparquet.write` Returns ------- bytes if no path argument is provided else None r"engine='auto' is deprecated and will be removed in a future version. Pass engine='pyarrow' or do not pass 'engine' to use the default.r)N)rFrrr@r) rHr/r rr0r1r rr8r#BytesIOrpgetvalue) rhrXr%rFrr@rrroimpl path_or_bufs r7 to_parquetrst.#&() S^^#&(8  + ')   f DAESWKDJJ   %'    |+rzz222##%%rjrc |tjur)|dk(r$tjdtt |tjurd}t |} t|| j|f||||||d|S)a) Load a parquet object from the file path, returning a DataFrame. This function requires the `pyarrow `_ library. The function automatically handles reading the data from a parquet file and creates a DataFrame with the appropriate structure. Parameters ---------- path : str, path object or file-like object String, path object (implementing ``os.PathLike[str]``), or file-like object implementing a binary ``read()`` function. The string could be a URL. Valid URL schemes include http, ftp, s3, gs, and file. For file URLs, a host is expected. A local file could be: ``file://localhost/path/to/table.parquet``. Certain URL schemes may require additional packages. For example, S3 URLs require the ``s3fs`` library. See :ref:`install.optional_dependencies` for a full list. A file URL can also be a path to a directory that contains multiple partitioned parquet files. Both pyarrow and fastparquet support paths to directories as well as file URLs. A directory path could be: ``file://localhost/path/to/tables`` or ``s3://bucket/partition_dir``. engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto' Parquet library to use. If 'auto', then the option ``io.parquet.engine`` is used. The default ``io.parquet.engine`` behavior is to try 'pyarrow', falling back to 'fastparquet' if 'pyarrow' is unavailable. When using the ``'pyarrow'`` engine and no storage options are provided and a filesystem is implemented by both ``pyarrow.fs`` and ``fsspec`` (e.g. "s3://"), then the ``pyarrow.fs`` filesystem is attempted first. Use the filesystem keyword with an instantiated fsspec filesystem if you wish to use its implementation. .. deprecated:: 3.1.0 The ``'fastparquet'`` and ``'auto'`` engine options are deprecated. Use ``'pyarrow'`` or do not pass ``engine`` to use the default. columns : list, default=None If not None, only these columns will be read from the file. storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc. For HTTP(S) URLs the key-value pairs are forwarded to ``urllib.request.Request`` as header options. For other URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more details, and for more examples on storage options refer `here `_. dtype_backend : {'numpy_nullable', 'pyarrow'} Back-end data type applied to the resultant :class:`DataFrame` (still experimental). If not specified, the default behavior is to not use nullable data types. If specified, the behavior is as follows: * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame` * ``"pyarrow"``: returns pyarrow-backed nullable :class:`ArrowDtype` :class:`DataFrame` .. versionadded:: 2.0 filesystem : fsspec or pyarrow filesystem, default None Filesystem object to use when reading the parquet file. Only implemented for ``engine="pyarrow"``. .. versionadded:: 2.1.0 filters : List[Tuple] or List[List[Tuple]], default None To filter out data. Filter syntax: [[(column, op, val), ...],...] where op is [==, =, >, >=, <, <=, !=, in, not in] The innermost tuples are transposed into a set of filters applied through an `AND` operation. The outer list combines these sets of filters through an `OR` operation. A single list of tuples can also be used, meaning that no `OR` operation between set of filters is to be conducted. Using this argument will NOT result in row-wise filtering of the final partitions unless ``engine="pyarrow"`` is also specified. For other engines, filtering is only performed at the partition level, that is, to prevent the loading of some row-groups and/or files. .. versionadded:: 2.1.0 to_pandas_kwargs : dict | None, default None Keyword arguments to pass through to :func:`pyarrow.Table.to_pandas` when ``engine="pyarrow"``. .. versionadded:: 3.0.0 **kwargs Additional keyword arguments passed to the engine: * For ``engine="pyarrow"``: passed to :func:`pyarrow.parquet.read_table` * For ``engine="fastparquet"``: passed to :meth:`fastparquet.ParquetFile.to_pandas` Returns ------- DataFrame DataFrame based on parquet file. See Also -------- DataFrame.to_parquet : Create a parquet object that serializes a DataFrame. Examples -------- >>> original_df = pd.DataFrame({"foo": range(5), "bar": range(5, 10)}) >>> original_df foo bar 0 0 5 1 1 6 2 2 7 3 3 8 4 4 9 >>> df_parquet_bytes = original_df.to_parquet() >>> from io import BytesIO >>> restored_df = pd.read_parquet(BytesIO(df_parquet_bytes)) >>> restored_df foo bar 0 0 5 1 1 6 2 2 7 3 3 8 4 4 9 >>> restored_df.equals(original_df) True >>> restored_bar = pd.read_parquet(BytesIO(df_parquet_bytes), columns=["bar"]) >>> restored_bar bar 0 5 1 6 2 7 3 8 4 9 >>> restored_bar.equals(original_df[["bar"]]) True The function uses `kwargs` that are passed directly to the engine. In the following example, we use the `filters` argument of the pyarrow engine to filter the rows of the DataFrame. Since `pyarrow` is the default engine, we can omit the `engine` argument. Note that the `filters` argument is implemented by the `pyarrow` engine, which can benefit from multithreading and also potentially be more economical in terms of memory. >>> sel = [("foo", ">", 2)] >>> restored_part = pd.read_parquet(BytesIO(df_parquet_bytes), filters=sel) >>> restored_part foo bar 0 3 8 1 4 9 r"rr))rsrr@rrr) r rr0r1r rr8rrt) rXr%rsr@rrrrrors r7 read_parquetr=s\S^^#&(8  + ')   f D & 499  '#)    rj)r%r/rur )Nr=F) rXz1FilePath | ReadBuffer[bytes] | WriteBuffer[bytes]r\rr@rr]r/r^boolruzVtuple[FilePath | ReadBuffer[bytes] | WriteBuffer[bytes], IOHandles[bytes] | None, Any])rhrrXz$FilePath | WriteBuffer[bytes] | Noner%str | lib.NoDefaultrFrrrr@rrrrrruz bytes | None)rXzFilePath | ReadBuffer[bytes]r%rrsrr@rrrrrrz:list[tuple[Any, ...]] | list[list[tuple[Any, ...]]] | Nonerrrur)8__doc__ __future__rr#rrWtypingrrrr0rrpandas._config.configr r+ pandas._libsr pandas.compat._optionalr pandas.errorsr r pandas.util._decoratorsrpandas.util._exceptionsrpandas.util._validatorsrrrpandas.io._utilrrpandas.io.commonrrrrrrpandas._typingrrrrrrr8rer r,r-rrrrzrjr7rs"   ;>/47%GV.2 ^' ;^' ^'+^'  ^'  ^'  ^'B ( (w (w tu hu t26"%..-5-1'+yy .y y+ y  y + y%yyyx H#&.. $-125..JN.2C &C CC+ C 0 C  CHC,CCCrj