qtj>KdZddlZddlZddlmZddlmZddlmZddl Z ddl Z ddl mZddlmZddlmZmZdd lmZdd lmZdd lmZd ed edefdZd ed ede jfdZdZ ddZ ddZ ddZ dS)z9Implementation of ARFF parsers: via LIAC-ARFF and pandas.N) OrderedDict) Generator)List)_arff)ArffSparseDataType)chunk_generatorget_chunk_n_rows)check_pandas_support)_align_api_if_sparse) pd_fillna arff_datainclude_columnsreturnctttf}dt|D}t|d|d|dD]a\}}}||vrW|d||d||d||b|S)aObtains several columns from sparse ARFF representation. Additionally, the column indices are re-labelled, given the columns that are not included. (e.g., when including [1, 2, 3], the columns will be relabelled to [0, 1, 2]). Parameters ---------- arff_data : tuple A tuple of three lists of equal size; first list indicating the value, second the x coordinate and the third the y coordinate. include_columns : list A list of columns to include. Returns ------- arff_data_new : tuple Subset of arff data with only the include columns indicated by the include_columns argument. ci|]\}}|| Sr.0 array_idx column_idxs [/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/sklearn/datasets/_arff_parser.py z)_split_sparse_columns../+"7)Z Ir)list enumeratezipappend)r r arff_data_newreindexed_columnsvalrow_idxcol_idxs r_split_sparse_columnsr&s.*.(@M;D_;U;U"%Yq\9Q<1!N!N@@Wg o % % !  # #C ( ( ( !  # #G , , , !  # #$5g$> ? ? ? rc@t|ddz}|t|f}dt|D}tj|tj}t |d|d|dD]\}}}||vr |||||f<|S)Nrci|]\}}|| Srrrs rrz)_sparse_data_to_array..Arrdtyperr)maxlenrnpemptyfloat64r) r rnum_obsy_shaper"yr#r$r%s r_sparse_data_to_arrayr3:s )A,!#GO,,-G;D_;U;U  +++A!$Yq\9Q<1!N!N99Wg o % %58Ag(11 2 Hrc||}t|dkr ||}n$t|dkr||d}nd}||fS)a Post process a dataframe to select the desired columns in `X` and `y`. Parameters ---------- frame : dataframe The dataframe to split into `X` and `y`. feature_names : list of str The list of feature names to populate `X`. target_names : list of str The list of target names to populate `y`. Returns ------- X : dataframe The dataframe containing the features. y : {series, dataframe} or None The series or dataframe containing the target. rrrN)r,)frame feature_names target_namesXr2s r_post_process_framer9Ls^, mA <A ,  \  a   ,q/ "  a4Krc "#$d}||}|dkr tjn tj}|dk } tj||| } ||z##fd| dD"|dkrt d} t | d} t | } t| d}| |g| d }| d  }t|}#fd | D}||g}t| d|D]3}|| || d |4t|dkr)|d|dj|d<| |d }t'| |}~~i}|jD]\}|d}|dkrd||<.|dkrd||<L|j|||<]||}t-|||\}$nI| d}fd|D}fd|D}t/|t0r|t3d|ddkrd}n|d|dz}t5jt8j|d|}|j|}|dd|f}|dd|f$nt/|t@rtC||}tE|ddz}|t|f} tFj$%|d|d|dff| t4j&}tO|(}tS||$nt3dtU|"fd |D}!|!sn\tW|!r/t5j,"$fd!t[|D$nt]|!rt3d"$j/ddkr$d#$n$j/ddkrd$|dkr|$|dfS|$d"fS)$aARFF parser using the LIAC-ARFF library coded purely in Python. This parser is quite slow but consumes a generator. Currently it is needed to parse sparse datasets. For dense datasets, it is recommended to instead use the pandas-based parser, although it does not always handles the dtypes exactly the same. Parameters ---------- gzip_file : GzipFile instance The file compressed to be read. output_arrays_type : {"numpy", "sparse", "pandas"} The type of the arrays that will be returned. The possibilities ara: - `"numpy"`: both `X` and `y` will be NumPy arrays; - `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array; - `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a pandas Series or DataFrame. columns_info : dict The information provided by OpenML regarding the columns of the ARFF file. feature_names_to_select : list of str A list of the feature names to be selected. target_names_to_select : list of str A list of the target names to be selected. Returns ------- X : {ndarray, sparse matrix, dataframe} The data matrix. y : {ndarray, dataframe, series} The target. frame : dataframe or None A dataframe containing both `X` and `y`. `None` if `output_array_type != "pandas"`. categories : list of str or None The names of the features that are categorical. `None` if `output_array_type == "pandas"`. c3BK|D]}|dVdS)Nutf-8)decode) gzip_filelines r_io_to_generatorz+_liac_arff_parser.._io_to_generators: ' 'D++g&& & & & & ' 'rsparsepandas) return_typeencode_nominalcNi|]!\}}t|tr|v||"Sr) isinstancer)rnamecatcolumns_to_selects rrz%_liac_arff_parser..sI D# c4 &*->%>%> c%>%>%>r attributeszfetch_openml with as_frame=TruedataF)columnscopyT)deepcg|]}|v| SrrrcolrIs r z%_liac_arff_parser..s$SSS3#AR:R:R3:R:R:Rrrrr) ignore_index data_typeintegerInt64nominalcategorycFg|]}t|dSindexintrcol_nameopenml_columns_infos rrRz%_liac_arff_parser..s<% % %  #H-g6 7 7% % % rcFg|]}t|dSrZr\r^s rrRz%_liac_arff_parser..s<$ $ $  #H-g6 7 7$ $ $ rNz6shape must be provided when arr['data'] is a Generatorr/)r*count)shaper*z-Unexpected type for data obtained from arff: ch|]}|vSrr)rr_ categoriess r z$_liac_arff_parser..s+   '/H "   rc g|]j\}}tjtj|ddd||dzft dkS)Or)NrF)rM)r-takeasarraypopastyper])rir_rfr2s rrRz%_liac_arff_parser..s $8 G :>>(#;#;3GGG!!!QQY,..s.??rzAMix of nominal and non-nominal targets is not currently supported)rb)0rCOO DENSE_GENloadr rrkeysnext DataFrame memory_usagesumr rr r,rmdtypesconcatr rLlowerr9rFr ValueErrorr-fromiter itertoolschain from_iterablereshapetupler&r+sprA coo_arrayr/r tocsrr3typeallhstackranyrd)%r>output_arrays_typer`feature_names_to_selecttarget_names_to_selectrdr@streamrCrDarff_containerpd columns_info column_names first_rowfirst_df row_bytes chunksizecolumns_to_keepdfsrKr5rwrG column_dtyper8r feature_indices_to_selecttarget_indices_to_selectrc arff_data_Xr0X_shapeis_classificationrfrIr2s% ` @@@r_liac_arff_parserrlsn''' i ( (F 2X==%))5?K-89NZKN02HH' 5J X%% !"C D D">,#?@@ L--//00 /00 << \<NN))t)4488:: $Y// TSSS,SSS()#N6$:IFF  D JJ T >?G ##Q+a.+a.!ABj$A %QWWYY//A%i1IJJAAQYQQ     3I   !   " # #   (11G'H'H AA" # # S  71:?? %  AA WQZ1__AX%%!UD  az !!rc "ddl|D]>}|ddrn?i|D]K}||d}|dkrd|<.|dkrd |<Lfd t |D} dd d gd d ddd| d } i| |pi}j|fi|} d|D| _n-#t$r } j d| d} ~ wwxYw||zfd| jD} | | } tj dfd}fd| j D}|D]%}| |j|| |<&t!| ||\}}|dkr||| dfS||}}fd| j D}||d|fS)a^ARFF parser using `pandas.read_csv`. This parser uses the metadata fetched directly from OpenML and skips the metadata headers of ARFF file itself. The data is loaded as a CSV file. Parameters ---------- gzip_file : GzipFile instance The GZip compressed file with the ARFF formatted payload. output_arrays_type : {"numpy", "sparse", "pandas"} The type of the arrays that will be returned. The possibilities are: - `"numpy"`: both `X` and `y` will be NumPy arrays; - `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array; - `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a pandas Series or DataFrame. openml_columns_info : dict The information provided by OpenML regarding the columns of the ARFF file. feature_names_to_select : list of str A list of the feature names to be selected to build `X`. target_names_to_select : list of str A list of the target names to be selected to build `y`. read_csv_kwargs : dict, default=None Keyword arguments to pass to `pandas.read_csv`. It allows to overwrite the default options. Returns ------- X : {ndarray, sparse matrix, dataframe} The data matrix. y : {ndarray, dataframe, series} The target. frame : dataframe or None A dataframe containing both `X` and `y`. `None` if `output_array_type != "pandas"`. categories : list of str or None The names of the features that are categorical. `None` if `output_array_type == "pandas"`. rNr<z@datarTrUrVrWrXc0i|]\}}|v ||Srr)rr%rGrws rrz'_pandas_arff_parser..s4 GT 6>>  >>rF?%"T\) header index_col na_valueskeep_default_nacomment quotecharskipinitialspace escapecharr*cg|]}|Srr)rrGs rrRz'_pandas_arff_parser..s>>>$>>>rzwThe number of columns provided by OpenML does not match the number of columns inferred by pandas when reading the file.cg|]}|v| SrrrPs rrRz'_pandas_arff_parser..s$PPPss>O7O7Os7O7O7Orz^'(?P.*)'$c`tj|}||S|dS)Ncontents)researchgroup) input_stringmatchsingle_quote_patterns rstrip_single_quotesz0_pandas_arff_parser..strip_single_quotess2 . == = {{:&&&rcDg|]\}}t|j|Sr)rFCategoricalDtyperrGr*rs rrRz'_pandas_arff_parser..s@ D% eR0 1 1 rrBcti|]4\}}t|j||j5Sr)rFrrftolistrs rrz'_pandas_arff_parser..sQ D% eR0 1 1 e%%''r)rBr=ry startswithrread_csvrLrzerrors ParserErrorrcompilerwitemsrHrename_categoriesr9to_numpy)r>rr`rrread_csv_kwargsr?rGrdtypes_positionaldefault_read_csv_kwargsr5excrrcategorical_columnsrQr8r2rfrIrwrrs @@@@r_pandas_arff_parserr8sp ;;w   % % ' ' 2 27 ; ;  E F#&&*40=     9 , ,#F4LL    ! !Y . .%F4L&':;;U  "  M0L_5JLO BK 5 5_ 5 5E  ?>*=>>> i## @    02HHPPPPemPPPO / "E:&=>>''''' <--// #KK3Z^556IJJc u&=?U V VDAqX%%!UD  zz||QZZ\\1 <--//J az !!sC++ D5DDc|dkrt||||||S|dkrt||||||Std|d)a6Load a compressed ARFF file using a given parser. Parameters ---------- gzip_file : GzipFile instance The file compressed to be read. parser : {"pandas", "liac-arff"} The parser used to parse the ARFF file. "pandas" is recommended but only supports loading dense datasets. output_type : {"numpy", "sparse", "pandas"} The type of the arrays that will be returned. The possibilities ara: - `"numpy"`: both `X` and `y` will be NumPy arrays; - `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array; - `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a pandas Series or DataFrame. openml_columns_info : dict The information provided by OpenML regarding the columns of the ARFF file. feature_names_to_select : list of str A list of the feature names to be selected. target_names_to_select : list of str A list of the target names to be selected. read_csv_kwargs : dict, default=None Keyword arguments to pass to `pandas.read_csv`. It allows to overwrite the default options. Returns ------- X : {ndarray, sparse matrix, dataframe} The data matrix. y : {ndarray, dataframe, series} The target. frame : dataframe or None A dataframe containing both `X` and `y`. `None` if `output_array_type != "pandas"`. categories : list of str or None The names of the features that are categorical. `None` if `output_array_type == "pandas"`. z liac-arffrBzUnknown parser: 'z%'. Should be 'liac-arff' or 'pandas'.)rrrz)r>parser output_typer`rrrdrs rload_arff_from_gzip_filersv    # "      8  "    # "      M M M M   r)N)NN)!__doc__r|r collectionsrcollections.abcrtypingrnumpyr-scipyrsklearn.externalsrsklearn.externals._arffrsklearn.utils._chunkingrr $sklearn.utils._optional_dependenciesr sklearn.utils._sparser sklearn.utils.fixesr r&ndarrayr3r9rrrrrrrs??  ######%%%%%%######666666EEEEEEEEEEEEEE666666)))))) ! 48     F ! 48 Z    $L  I"I"I"I"d U"U"U"U"~ P P P P P P r