
     hPv                    p   d dl mZ d dlmZ d dlmZ d dlmZ d dlZd dlZd dlZ	d dl
mZ d dlmZ d dlZd dlmZ d d	lmZ d d
lmZ d dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z# d dl"m$Z$ d dl"m%Z% d dl"m&Z& d dl"m'Z' d dl"m(Z( d dl"m)Z) d dl"m*Z* d dl"m+Z+ d dl"m,Z, d dl"m-Z- d dl"m.Z. d d l"m/Z/ d d!l"m0Z0 d d"l"m1Z1 d d#l"m2Z2 d d$l"m3Z3 d d%l"m4Z4 d d&l"m5Z5 d d'l"m6Z6 d d(l7m8Z8 d d)l9m:Z: d d*l;m<Z= dd,Z>d- Z?d. Z@ejA        B                    d/g d0          d1             ZCd2 ZDd3 ZEed4             ZFed5             ZGed6             ZHd7 ZId8 ZJd9 ZKed:             ZLd; ZMd< ZNd= ZOd> ZPd? ZQd@ ZRejA        B                    dAg dB          dC             ZSdD ZTdE ZUejA        B                    dF e	jV        g dG           e	jV        g dG          dHdIf e	jV        g dJ           e	jV        g dJ          dHdKf e	jV        g dJ           e	jV        g dG          dHdLf e	jV        g dJ           e	jV        g dM          dHdNf e	jV        g dG           e	jV        g dJ          dHdOfg          dP             ZWejA        B                    dQ e	jV        g dR           e	jV        g dS          dHdTfg          dU             ZXdV ZYdW ZZdX Z[dY Z\dZ Z]d[ Z^d\ Z_ejA        B                    d]d^d_g          d`             Z`da ZaejA        B                    dbg dc          dd             Zbde Zcdf Zddg ZeejA        B                    dhg difdjdkgdlfgdmdngo          dp             ZfejA        B                    dqdd drgg dsfg dto          du             Zgdv ZhejA        B                    dwg dx          dy             Zidz Zjd{ Zkd| Zld} Zmd~ Znd Zod Zpd Zqd Zred             Zsd Ztd Zud Zvd Zwd Zxd Zyd ZzejA        B                    dddg          d             Z{ed             Z|ed             Z}eejA        B                    d/g d0          d                         Z~ejA        B                    ddrg          ejA        B                    dbg d          ejA        B                    d/d drg          d                                     ZejA        B                    dbg d          d             ZejA        B                    d/d drg          d             Zd Zd ZejA        B                    d/d drg          d             ZejA        B                    d/g d0          d             ZejA        B                    d/g d0          d             ZejA        B                    d/g d0          d             Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd ZejA        B                    dg dg dfg dg dfg dg dfg          d             ZdS )    )partial)product)chain)permutationsN)linalg)	bernoulli)datasets)svm)make_multilabel_classification)label_binarizeLabelBinarizer)check_random_state)assert_almost_equal)assert_array_equal)assert_array_almost_equal)assert_allclose)assert_no_warnings)ignore_warnings)MockDataFrame)accuracy_scoreaverage_precision_score)balanced_accuracy_score)class_likelihood_ratiosclassification_report)cohen_kappa_score)confusion_matrix)f1_score)fbeta_score)hamming_loss)
hinge_loss)jaccard_score)log_lossmatthews_corrcoef)precision_recall_fscore_support)precision_score)recall_score)zero_one_loss)brier_score_loss)multilabel_confusion_matrix_check_targets)UndefinedMetricWarning)hammingFc                    | t          j                    } | j        }| j        }|r||dk              ||dk              }}|j        \  }}t          j        |          }t          d          }|                    |           ||         ||         }}t          |dz            }t
          j
                            d          }t
          j        ||                    |d|z            f         }t          j        ddd          }	|	                    |d|         |d|                                       ||d                   }
|r|
ddd	f         }
|	                    ||d                   }||d         }|||
fS )
zMake some classification predictions on a toy dataset using a SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    N   %   r      linearT)kernelprobabilityrandom_state   )r	   	load_irisdatatargetshapenparanger   shuffleintrandomRandomStatec_randnr
   SVCfitpredict_probapredict)datasetbinaryXy	n_samples
n_featuresprnghalfclfprobas_predy_predy_trues                e/var/www/html/Sam_Eipo/venv/lib/python3.11/site-packages/sklearn/metrics/tests/test_classification.pymake_predictionrX   9   sz    $&&AA "Qx1q51GIz
	)A
R
 
 CKKNNNQ41qAy1}D )


"
"C
a9cJ&67778A 't!
D
D
DC''!ETE(AeteH--;;AdeeHEEK ( "!!!Q$'[[455""FtuuXF6;&&    c            
         t          j                    } t          | d          \  }}}ddddddd	d
ddddddddddddddddddd}t          ||t	          j        t          | j                            | j        d          }|                                |                                k    sJ |D ]}|dk    r2t          ||         t                    sJ ||         ||         k    sJ :||                                         ||                                         k    sJ ||         D ]*}t          ||         |         ||         |                    +t          |d         d                   t          k    sJ t          |d         d                   t          k    sJ t          |d         d                   t          k    sJ t          |d         d                   t          k    sJ d S ) NFrJ   rK   g7Mo?gUUUUUU?ghQ?   )	precisionrecallf1-scoresupportUUUUUU?gc1Ƹ?g433333?   g)k??$I$I?   gCFQ?gc?gƢ?K   r_   r]   r^   r`   g?gD~WG?g]3p?)setosa
versicolor	virginica	macro avgaccuracyweighted avgT)labelstarget_namesoutput_dictrl   rh   r]   rk   r`   )r	   r:   rX   r   r>   r?   lenro   keys
isinstancefloatr   typerA   )irisrV   rU   _expected_reportreportkeymetrics           rW   ,test_classification_report_dictionary_outputr|   i   sr    D'UCCCFFA
 -)*	
 
 -*+	
 
 -)+	
 
 ++'	
 
 '++(	
 
5   OD #yT.//00&  F ;;==O00222222 W W*fSk511111#;/#"666666#;##%%)=)B)B)D)DDDDD)#. W W#OC$8$@&+fBUVVVVW )+6775@@@@,[9::eCCCC))455<<<<,Y788C??????rY   c                  \   t          g g d          } dt          j        t          j        t          j        dddddddd}t          | t                    sJ |                                 |                                k    sJ |D ]}|dk    r2t          | |         t                    sJ | |         ||         k    sJ :| |                                         ||                                         k    sJ ||         D ]*}t          ||         |         | |         |                    +d S )NT)rV   rU   rp           r   rg   )rl   rk   rm   rl   )r   r>   nanrs   dictrr   rt   r   )ry   rx   rz   r{   s       rW   2test_classification_report_output_dict_empty_inputr      si   ""RTJJJFf	
 
 	
 
 O fd#####;;==O00222222 W W*fSk511111#;/#"666666#;##%%)=)B)B)D)DDDDD)#. W W#OC$8$@&+fBUVVVVWW WrY   zero_division)warnr   r9   c                    g dg d}}t          j        d          5 }t          ||| d           | dk    r5t          |          dk    sJ |D ]}d}|t	          |j                  v sJ n|rJ d d d            d S # 1 swxY w Y   d S )	Nabc)r   r   dTrecord)r   rp   r   r9   z7Use `zero_division` parameter to control this behavior.)warningscatch_warningsr   rq   strmessage)r   rV   rU   r   itemmsgs         rW   0test_classification_report_zero_division_warningr      s   $__oooFF		 	-	-	- 
F-T	
 	
 	
 	
 F""v;;???? 0 0Oc$,///////0 :
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   AA>>BBc                  F   t          j        g dg dg          } t          j        g dg dg          }t          | |          dk    sJ t          | |           dk    sJ t          ||          dk    sJ t          |t          j        |                    dk    sJ t          | t          j        |                     dk    sJ t          | t          j        | j                            dk    sJ t          |t          j        | j                            dk    sJ d S )Nr   r9   r9   r9   r   r9   r   r   r9         ?r9   r   )r>   arrayr   logical_notzerosr=   y1y2s     rW   .test_multilabel_accuracy_score_subset_accuracyr      s)   	999iii(	)	)B	999iii(	)	)B"b!!S(((("b!!Q&&&&"b!!Q&&&&"bnR0011Q6666"bnR0011Q6666"bhrx0011Q6666"bhrx0011Q666666rY   c                  L   t          d          \  } }}t          | |d           \  }}}}t          |ddgd           t          |ddgd           t          |d	d
gd           t          |ddg           i t          fddit          ffD ]\  }} |t
          | |fi |}	t          |	dd            |t          | |fi |}
t          |
dd            |t          | |fi |}t          |d
d           t           |t          | |fddi|d|	z  |
z  d|	z  |
z   z  d           d S )NTrK   averageg\(\?g333333?r2   g)\(?g(\?皙?gRQ?   r   rK   beta      )
rX   r'   r   r   r   r(   r)   r   r   r    )rV   rU   rw   rP   rfskwargs	my_assertpsrsfss               rW   %test_precision_recall_f1_score_binaryr      s   't444FFA 1NNNJAq!Qa$q111a$q111a$q111q2r(### 
 
X	 23 
 
	 YAA&AA!"dA...Y|VV>>v>>!"dA...Yx::6::!"dA...Ik66DDDVDDOb D2IN3	
 	
 	
 	

 
rY   c                  f   dt          ddgddg          k    sJ dt          ddgddg          k    sJ dt          ddgddg          k    sJ dt          ddgddgd          k    sJ dt          ddgddg          k    sJ dt          ddgddg          k    sJ dt          ddgddg          k    sJ dt          ddgddgt	          d                    k    sJ t          ddgddgt	          d                    t          j        t          ddgddgd                    k    sJ d S )	N      ?r9   r   r   r~   infg     j@)r(   r)   r   r    rt   pytestapprox rY   rW   +test_precision_recall_f_binary_single_classr   	  s   
 /1a&1a&111111,1v1v......(Aq6Aq6******+q!fq!f15555555/2r(RH555555,Bx"b222222(B8b"X......+r2hRuU||DDDDDDDBx"be===RHr2hS111B B      rY   c                     g d} g d}t          | t          j        d                    }t          |t          j        d                    }| |f||fg}t          |          D ]\  }\  } }t	          | |g dd           }t          g d|           t	          | |g dd          }t          t          j        g d          |           d	D ]C}|d
k    r|dk    rt          t	          | |g d|          t	          | |d |                     DdD ]}t          j	        t                    5  t	          ||t          j        d          |           d d d            n# 1 swxY w Y   t          j	        t                    5  t	          ||t          j        dd          |           d d d            n# 1 swxY w Y   t          j        g dg dg          } t          j        g dg dg          }t          | |d
ddg          \  }}	}
}t          t          j        ||	|
g          t          j        g d                     d S )N)r9      r   r2   )r9   r9   r   r2   r   classes)r   r9   r2   r   r   rn   r   )r~   r   r   r   r~   macro)microweightedsamplesr   r   )Nr   r   r      r   r   r   r9   r   r   r9   r9   r9   r   r9   r   rn   )      ?r9   竪?)r   r>   r?   	enumerater)   r   meanr   r   raises
ValueErrorr   r'   )rV   rU   
y_true_bin
y_pred_binr;   iactualr   rP   r   r   rw   s               rW   $test_precision_recall_f_extra_labelsr     s.    \\F\\F	!===J	!===JVz:67D(  FFff___dSSS!";";";VDDD ff___gVVV!"'*C*C*C"D"DfMMM 8 	 	G)##QVVOOOWUUUVVD'JJJ   	 7  ]:&& 	W 	WZ	!gVVVV	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W]:&& 	 	JryQ/?/?   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 Xyyy))),--FXyyy))),--F0	1a&  JAq!Q !Q++RX6G6G6G-H-HIIIIIs$   2&E$$E(	+E(	'F;;F?	F?	c                  \   g d} g d}t          | t          j        d                    }t          |t          j        d                    }| |f||fg}t          |          D ]\  }\  } }t	          t
          | |ddg          }t	          t
          | |d           }t          dd	g |d 
                     t          d |d
                     t          d |d
                     t          d |d
                     dD ]} ||
           ||
          k    sJ d S )N)r9   r9   r2   r   )r9   r   r   r   r   r   r9   r   rn   r   r   r   r   r   UUUUUU?r   r   )r   r   r   )r   r>   r?   r   r   r)   r   r   )	rV   rU   r   r   r;   r   	recall_13
recall_allr   s	            rW   &test_precision_recall_f_ignored_labelsr   J  s{    \\F\\F	!===J	!===JVz:67D( M MFFL&&!QHHH	\66$GGG
!3*ii.E.E.EFFFOYYw-G-G-GHHH3YYz5R5R5RSSSGYYw%?%?%?@@@ 6 	M 	MG9W---G1L1L1LLLLLL	MM MrY   c                  
   t          d          } |                     d          }|                     ddd          }d}t          j        t
          |          5  t          ||           d d d            d S # 1 swxY w Y   d S )Ni  
   r   r   sizez"multiclass format is not supportedmatch)r   randrandintr   r   r   r   )rQ   rU   rV   err_msgs       rW   3test_average_precision_score_score_non_binary_classr   a  s     S
!
!CXXb\\F [[AB[''F2G	z	1	1	1 0 0///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   A88A<?A<c                  B    g d} g d}t          | |          dk    sJ d S )N)r   r   r   r   r9   r9   r9   r9   r9   r9   r9   )r   皙?r   皙?r   333333?r   rc   rc   r9   r9   r9   r   rV   y_scores     rW   -test_average_precision_score_duplicate_valuesr   n  s;     /..F???G"6733q888888rY   c                  B    g d} g d}t          | |          dk    sJ d S )Nr   )r   r   r   r   r   r   s     rW   (test_average_precision_score_tied_valuesr   y  s7     YYFooG"6733s::::::rY   c                     t          d          \  } }}t          j        t                    5  t	          | |d           d d d            n# 1 swxY w Y   t          j        t                    5  t	          | |dd           d d d            n# 1 swxY w Y   t          j        t                    5  t	          g dg d	d
           d d d            d S # 1 swxY w Y   d S )NTr   gr   r2   rK   	pos_labelr   r   r9   r2   r9   r2   r   megar   )rX   r   r   r   r'   )rV   rU   rw   s      rW   +test_precision_recall_fscore_support_errorsr     s   't444FFA 
z	"	" C C'TBBBBC C C C C C C C C C C C C C C 
z	"	" W W'!XVVVVW W W W W W W W W W W W W W W 
z	"	" N N'			999fMMMMN N N N N N N N N N N N N N N N N Ns5   AAA1BBB5CC Cc                      d} t          j        t          |           5  t          g dg ddd           d d d            d S # 1 swxY w Y   d S )NzNote that pos_label \(set to 2\) is ignored when average != 'binary' \(got 'macro'\). You may use labels=\[pos_label\] to specify a single positive class.r   )r9   r2   r9   r9   r2   r2   r2   r   r   )r   warnsUserWarningr'   r   s    rW   (test_precision_recall_f_unused_pos_labelr     s    
	  
k	-	-	- 
 
'IIyyyAw	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   AA
Ac                      t          d          \  } }}d } || |            |d | D             d |D                        d S )NTr   c                 n   t          | |          }t          |ddgddgg           |                                \  }}}}||z  ||z  z
  }t          j        ||z   ||z   z  ||z   z  ||z   z            }|dk    rdn||z  }	t          | |          }
t          |
|	d           t          |
dd           d S )	N   r         r   r2   decimal=
ףp=?)r   r   flattenr>   sqrtr&   r   )rV   rU   cmtpfpfntnnumdentrue_mccmccs              rW   testz*test_confusion_matrix_binary.<locals>.test  s    ff--2Q!R1222BB2gRgrBw27+rBw727CDDq11cCi//!#x;;;;!#tQ777777rY   c                 ,    g | ]}t          |          S r   r   .0rM   s     rW   
<listcomp>z0test_confusion_matrix_binary.<locals>.<listcomp>      	!	!	!Q#a&&	!	!	!rY   c                 ,    g | ]}t          |          S r   r  r  s     rW   r  z0test_confusion_matrix_binary.<locals>.<listcomp>      #;#;#;qCFF#;#;#;rY   rX   rV   rU   rw   r  s       rW   test_confusion_matrix_binaryr    sr    't444FFA8 8 8 	DD	!	!&	!	!	!#;#;F#;#;#;<<<<<rY   c                      t          d          \  } }}d } || |            |d | D             d |D                        d S )NTr   c                 b    t          | |          }t          |ddgddggddgddggg           d S )Nr   r   r   r   r,   r   )rV   rU   r  s      rW   r  z5test_multilabel_confusion_matrix_binary.<locals>.test  sF    (882"a1b' 2b!Wq"g4FGHHHHHrY   c                 ,    g | ]}t          |          S r   r  r  s     rW   r  z;test_multilabel_confusion_matrix_binary.<locals>.<listcomp>  r  rY   c                 ,    g | ]}t          |          S r   r  r  s     rW   r  z;test_multilabel_confusion_matrix_binary.<locals>.<listcomp>  r  rY   r  r  s       rW   'test_multilabel_confusion_matrix_binaryr    su    't444FFAI I I 	DD	!	!&	!	!	!#;#;F#;#;#;<<<<<rY   c                      t          d          \  } }}dd} || |            |d | D             d |D             d           d S )	NFr   c           	         t          | |          }t          |ddgddggddgddggd	d
gddggg           |rg dng d}t          | ||          }t          |ddgddggd	d
gddggddgddggg           |rg dng d}t          | ||          }t          |ddgddggd	d
gddggddgddggddgddggg           d S )N/   r   r      &   r      r      r   r2      )021r   r2   r9   r   )r%  r&  r'  3)r   r2   r9   r   rf   r   r  )rV   rU   string_typer  rn   s        rW   r  z9test_multilabel_confusion_matrix_multiclass.<locals>.test  s[   (882q'Ar7#r1gAw%72r(QG9LM	
 	
 	

 %0>YYY(GGG2q'Ar7#r2hB%8B7RG:LM	
 	
 	

 *5F%%%%,,,(GGGa1b'"bAr7#a2q'"a1a&!		
 	
 	
 	
 	
rY   c                 ,    g | ]}t          |          S r   r  r  s     rW   r  z?test_multilabel_confusion_matrix_multiclass.<locals>.<listcomp>  r  rY   c                 ,    g | ]}t          |          S r   r  r  s     rW   r  z?test_multilabel_confusion_matrix_multiclass.<locals>.<listcomp>  r  rY   T)r*  )Fr  r  s       rW   +test_multilabel_confusion_matrix_multiclassr-    s|    'u555FFA
 
 
 
6 	DD	!	!&	!	!	!#;#;F#;#;#;NNNNNNrY   c                  Z   ddl m} m} t          j        g dg dg dg          }t          j        g dg dg dg          } ||          } ||          } | |          } | |          }t          j        g d	          }d
dgd
d
ggd
dgd
d
ggddgd
dggg}	|||g}
|||g}|
D ]'}|D ]"}t          ||          }t          ||	           #(t          ||d          }t          |d
dgd
d
ggd
d
gdd
ggdd
gddggg           t          ||ddg          }t          |ddgd
dggd
dgd
d
ggg           t          ||ddgd          }t          |ddgd
d
ggd
d
gddggdd
gd
dggg           t          |||d          }t          |ddgddggd
d
gdd
ggddgddggg           d S )Nr   )
csc_matrix
csr_matrixr   r   r9   r   r9   r9   r   r   r   r   )r2   r9   r   r9   r2   T
samplewiser   )rn   r4  )sample_weightr4  r   r   )scipy.sparser/  r0  r>   r   r,   r   )r/  r0  rV   rU   
y_true_csr
y_pred_csr
y_true_csc
y_pred_cscr5  real_cmtruespreds
y_true_tmp
y_pred_tmpr  s                  rW   +test_multilabel_confusion_matrix_multilabelr@    s   33333333Xyyy)))YYY788FXyyy)))YYY788FF##JF##JF##JF##J HYYY''MAA1a&1a&!1QFQF3CDGZ,EZ,E , ,
 	, 	,J,ZDDBr7++++	,
 
%VV	E	E	EBraVaV,1v1v.>!Q!Q@PQRRR 
%VVQF	C	C	CBraVaV,1v1v.>?@@@ 
%VVQFt	T	T	TBraVaV,1v1v.>!Q!Q@PQRRR 
%m
 
 
B raVaV,1v1v.>!Q!Q@PQRRRRRrY   c            	         t          j        g dg dg dg          } t          j        g dg dg dg          }t          j        t          d          5  t          | |d	d
g           d d d            n# 1 swxY w Y   t          j        t          d          5  t          | |g dg dg dg           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          | |dg           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          | |dg           d d d            n# 1 swxY w Y   t          j        t          d          5  t          g dg dd           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          g dg dgg dg dg           d d d            d S # 1 swxY w Y   d S )Nr   r1  r2  r   r   r   zinconsistent numbers of samplesr   r9   r2   r5  zshould be a 1d arrayr9   r2   r   )r2   r   r   )r   r   r   z%All labels must be in \[0, n labels\)r   r   r   zSamplewise metricsr   r   Tr3  z'multiclass-multioutput is not supported)r2   r9   r   r9   r   r2   )r>   r   r   r   r   r,   )rV   rU   r   s      rW   'test_multilabel_confusion_matrix_errorsrE    s   Xyyy)))YYY788FXyyy)))YYY788F 
z)J	K	K	K J J#FF1a&IIIIJ J J J J J J J J J J J J J J	z)?	@	@	@ 
 
#F999iii*K	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 7G	z	1	1	1 A A#FFB4@@@@A A A A A A A A A A A A A A A6G	z	1	1	1 @ @#FFA3????@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ 
z)=	>	>	> K K#IIIyyyTJJJJK K K K K K K K K K K K K K K 8G	z	1	1	1 T T#YYY			$:YYY			<RSSST T T T T T T T T T T T T T T T T Tsl   A77A;>A;CC	C	-DDD5EEE;FF"%F"G00G47G4z%normalize, cm_dtype, expected_results))truer   TUU?)predr   rG  )allr   geq?)Nr   r2   c                     g ddz  }t          t          t          g d                     }t          |||           }t	          ||           |j        j        |k    sJ d S )Nr   r   	normalize)listr   r   r   r   dtypekind)rL  cm_dtypeexpected_resultsy_testrU   r  s         rW   test_confusion_matrix_normalizerS  2  so     YY]F%iii00122F	&&I	>	>	>BB()))8=H$$$$$$rY   c                      g d} g d}t          j        t          d          5  t          | |d           d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   r9   r9   r9   r9   r   r   r   r   r   r   r   r   znormalize must be one ofr   TrK  )r   r   r   r   )rR  rU   s     rW   ,test_confusion_matrix_normalize_wrong_optionrW  C  s    %%%F%%%F	z)C	D	D	D 9 9488889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9s   AAAc                  H   g d} g d}t          | |d          }|                                t          j        d          k    sJ t	          j                    5  t	          j        dt                     t          | |d          }d d d            n# 1 swxY w Y   |                                t          j        d          k    sJ t	          j                    5  t	          j        dt                     t          || d           d d d            d S # 1 swxY w Y   d S )	NrU  rV  rF  rK         @errorrH  r   )r   sumr   r   r   r   simplefilterRuntimeWarning)rR  rU   cm_truecm_preds       rW   ,test_confusion_matrix_normalize_single_classr`  J  s   %%%F%%%Fvv@@@G;;==FM#...... 
	 	"	" E Eg~666"66VDDDE E E E E E E E E E E E E E E ;;==FM#......		 	"	" ; ;g~6666::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;s$   -BBB-DDDzparams, warn_msg)r   r   r   r   r   r   rV   rU   z2samples of only one class were seen during testing)r9   r9   r9   r   r   r   z:positive_likelihood_ratio ill-defined and being set to nanz+no samples predicted for the positive class)r   r   r   r9   r9   r9   z:negative_likelihood_ratio ill-defined and being set to nanz@no samples of the positive class were present in the testing setc                     t          j        t          |          5  t          di |  d d d            d S # 1 swxY w Y   d S Nr   r   )r   r   r   r   )paramswarn_msgs     rW   test_likelihood_ratios_warningsrf  ]  s    b 
k	2	2	2 * *))&)))* * * * * * * * * * * * * * * * * *   6::zparams, err_msg)r   r9   r   r9   r   r9   r9   r   r   r2   zeclass_likelihood_ratios only supports binary classification problems, got targets of type: multiclassc                     t          j        t          |          5  t          di |  d d d            d S # 1 swxY w Y   d S rc  )r   r   r   r   )rd  r   s     rW   test_likelihood_ratios_errorsrj    s      
z	1	1	1 * *))&)))* * * * * * * * * * * * * * * * * *rg  c                  F   t          j        dgdz  dgdz  z             } t          j        dgdz  dgdz  z   dgdz  z             }t          | |          \  }}t          |d           t          |d	           t          | |           \  }}t	          |t           j        dz             t          |t          j        d          d
           t          j        dgdz  dgdz  z             }t          | ||          \  }}t          |d           t          |d           d S )Nr9   r   r   r   r2   r   r   g?g_B{	%?g-q=)rtolr      r~   r   rB  gUUUUUU@gqq?)r>   r   r   r   r   r   r   )rV   rU   posnegr5  s        rW   test_likelihood_ratiosrp    s*    XqcAgb())FXqcAgb(A37233F&vv66HCC!!!C!!! 'vv66HCsBFQJ'''C!51111
 HcURZ3%!)344M&vv]SSSHCC   C!!!!!rY   c                     t          j        dgdz  dgdz  z             } t          j        dgdz  dgdz  z   dgdz  z   dgdz  z             }t          | |          }t          |dd	
           |t          ||           k    sJ t          j        | dgdz            } t          j        |dgdz            }t          | |ddg          |k    sJ t          t          | |           d           t          j        dgdz  dgdz  z   dgdz  z             } t          j        dgdz  dgdz  z   dgdz  z             }t          t          | |          dd
           t          j        dgdz  dgdz  z   dgdz  z             } t          j        dgdz  dgdz  z   dgdz  z             }t          t          | |          dd
           t          t          | |d          dd
           t          t          | |d          dd
           d S )Nr   (   r9   <   re   r   2   gʡE?r   r   r2   r   r   r   .   ,   4          g??g+?r5   weightsg_vO?	quadraticg#?)r>   r   r   r   append)r   r   kappas      rW   test_cohen_kappar    sV    
1#(aS2X%	&	&B	1#(aS2X%b0A38;	<	<Bb"%%Eua0000%b"------ 
2sQw		B	2sQw		BRQF333u<<<<)"b113777 
1#(aS2X%b0	1	1B	1#(aS2X%b0	1	1B)"b1161EEEE 
1#(aS2X%b0	1	1B	1#(aS2X%b0	1	1B)"b1161EEEE)"b(CCCVUVWWWW"b+666     rY   c                  j    t          dgdg          dk    sJ t          ddgddg          dk    sJ d S )Nr   r9   r~   r%   r   rY   rW   test_matthews_corrcoef_nanr    sJ    aS1#&&#----aVaV,,333333rY   c                     t           j                            d          } |                     ddd          }|                     ddd          }t	          t          ||          t          j        ||          d         d           d S )Nr   r2   re   r   r   r9   r   )r>   rB   rC   r   r   r&   corrcoef)rQ   rV   rU   s      rW   -test_matthews_corrcoef_against_numpy_corrcoefr    s    
)


"
"C[[AB[''F[[AB[''F&&))2;vv+F+Ft+Lb    rY   c                    	
 t           j                            d          } |                     ddd          }|                     ddd          }|                     d          }t          |||          	t          	          
t          	
fdt          
          D                       }t          	
fdt          
          D                       }t          j        	
fdt          
          D                       }|t          j	        ||z            z  }t          |||          }t          ||d	           d S )
Nr   r2   re   r   rB  c           	          g | ]Q}t                    D ]?}t                    D ]-}||f         ||f         z  ||f         ||f         z  z
  .@RS r   range)r  kmlCNs       rW   r  z9test_matthews_corrcoef_against_jurman.<locals>.<listcomp>  s     	
 	
 	
1XX	
 	
 1XX		
 	
  adGa1g!Q$!AqD' 11	
 	
 	
 	
 	
rY   c           	          g | ]Pd d f                                          t          j         fdt                    D                       z  QS )Nc                 T    g | ]$}t                    D ]}|k    ||f         %S r   r  r  r   gr  r  r  s      rW   r  zDtest_matthews_corrcoef_against_jurman.<locals>.<listcomp>.<listcomp>  8    LLL!qLLAQ!VVa1gVVVVrY   r[  r>   r  r  r  r  r  s    @rW   r  z9test_matthews_corrcoef_against_jurman.<locals>.<listcomp>  st     	
 	
 	
  aaadGKKMMfLLLLLLuQxxLLLMMN	
 	
 	
rY   c           	          g | ]Pd d f                                          t          j         fdt                    D                       z  QS )Nc                 T    g | ]$}t                    D ]}|k    ||f         %S r   r  r  s      rW   r  zDtest_matthews_corrcoef_against_jurman.<locals>.<listcomp>.<listcomp>
  r  rY   r  r  s    @rW   r  z9test_matthews_corrcoef_against_jurman.<locals>.<listcomp>  st     	
 	
 	
  adGKKMMfLLLLLLuQxxLLLMMN	
 	
 	
rY   r   )r>   rB   rC   r   r   r   rq   r[  r  r  r&   r   )rQ   rV   rU   r5  cov_ytypcov_ytytcov_ypyp
mcc_jurmanmcc_oursr  r  s            @@rW   %test_matthews_corrcoef_against_jurmanr    s    )


"
"C[[AB[''F[[AB[''FHHRLLM}EEEAAA	
 	
 	
 	
 	
1XX	
 	
 	
 H 	
 	
 	
 	
 	
 1XX	
 	
 	
 H v	
 	
 	
 	
 	
 1XX	
 	
 	
 H BGHx$7888J }MMMH*b11111rY   c            	      R   t           j                            d          } d |                     ddd          D             }t	          t          ||          d           d |D             }t	          t          ||          d           t          |d	d
g          }t          j        |d	d
          }t	          t          ||          d           t	          t          g dg d          d           t	          t          |d	gt          |          z            d           g d}g d}t	          t          ||          d           dgdz  dgdz  z   }t          j
        t                    5  t	          t          |||          d           d d d            d S # 1 swxY w Y   d S )Nr   c                 "    g | ]}|d k    rdndS )r   r   r   r   r  r   s     rW   r  z*test_matthews_corrcoef.<locals>.<listcomp>  s$    IIIQ!VVccIIIrY   r2   re   r   r   c                 "    g | ]}|d k    rdnd S )r   r   r   r  s     rW   r  z*test_matthews_corrcoef.<locals>.<listcomp>  s$    ;;;qc##s;;;rY   r   r   r   r   r   r   r   r   r~   )r9   r   r9   r9   r   r9   r9   r9   r   r9   r9   r9   r9   r9   r9   r9   r   r9   r9   r9   )r9   r9   r9   r   r   r9   r9   r9   r9   r   r9   r9   r9   r   r9   r9   r9   r   r9   r9   r9   r   rB  )r>   rB   rC   r   r   r&   r   whererq   r   r   AssertionError)rQ   rV   
y_true_invy_true_inv2y_1y_2masks          rW   test_matthews_corrcoefr    s   
)


"
"CIIckk!QRk.H.HIIIF )&&993??? <;F;;;J)&*==rBBB #s<<<K(;S11K)&+>>CCC ),,,EEsKKK )&3%#f++2EFFLLL G
F
FC
F
F
FC)#s33S999 38qcBhD 
~	&	& R R-c3dKKKSQQQR R R R R R R R R R R R R R R R R Rs   .!FF #F c                  B  	 t           j                            d          } t          d          	d}	fd|                     d|d          D             }t          t          ||          d           g d}g d	}t          t          ||          d
           g d}g d}t          t          ||          dt          j        d          z             g d}g d}t          t          ||          d           g d}g d}t          t          ||          d           g d}g d}t          t          ||          d           g d}g d}g d}t          t          |||          d           g d}g d}g d}t          t          |||          d           d S )Nr   r   r   c                 4    g | ]}t          |z             S r   )chr)r  r   ord_as     rW   r  z5test_matthews_corrcoef_multiclass.<locals>.<listcomp>;  s#    IIIc%!)nnIIIrY   re   r   r   )r   r   r9   r9   r2   r2   )r2   r2   r   r   r9   r9   g      )r9   r9   r   r   r   r   ii  r   )r   r   r   r~   	r   r9   r2   r   r9   r2   r   r9   r2   )	r9   r9   r9   r2   r2   r2   r   r   r   )r   r   r9   r9   r2   rh  r9   r9   r9   r9   r   rB  r   )r   r   r9   r2   r9   r9   r   r   )r>   rB   rC   ordr   r   r&   r  )
rQ   	n_classesrV   
y_pred_bad
y_pred_minrU   r  r  r5  r  s
            @rW   !test_matthews_corrcoef_multiclassr  7  s   
)


"
"CHHEIIIIIckk!YRk&H&HIIIF )&&993???  F###J)&*==tDDD  F###J)&*==sRWWEUEU?UVVV YYFYYF)&&993??? YYFYYF)&&993??? &
%
%C
%
%
%C)#s33S999 __F__F#OOM&&FFF   \\F\\F LLM&&FFF    rY   n_pointsd   i'  c                    t           j                            d          d }fd}t          j        ddg|           }t	          t          ||          d           t          j        g d|           }t	          t          ||          d            ||           \  }}t	          t          ||          d           t	          t          ||           |||                     d S )Ni3c                     t          | |          }|d         }|d         }|d         }t          |           }||z   |z  }||z   |z  }||z  ||z  z
  }	||z  d|z
  z  d|z
  z  }
|	t          j        |
          z  S )Nr9   r9   )r9   r   r  r9   )r   rq   r>   r  )rV   rU   conf_matrixtrue_pos	false_pos	false_negr  pos_rateactivitymcc_numeratormcc_denominators              rW   mcc_safez1test_matthews_corrcoef_overflow.<locals>.mcc_safet  s    &vv66t$%	%	v;;y(H4y(H4 8+h.AA"X-X>!h,Orw7777rY   c                                          |           }|d                     |           dz
  z  z   }|dk    }|dk    }||fS )N皙?r   )random_sample)r  x_truex_predrV   rU   rQ   s        rW   	random_ysz2test_matthews_corrcoef_overflow.<locals>.random_ys  sT    ""8,,#!2!28!<!<s!BCC##v~rY   r~   r   )r~   r   rY  )r>   rB   rC   repeatr   r&   )r  r  r  arrrV   rU   rQ   s         @rW   test_matthews_corrcoef_overflowr  o  s     )


)
)C
8 
8 
8     )S#J
)
)C)#s33S999
)OOOX
.
.C)#s33S999Yx((NFF)&&993???)&&9988FF;S;STTTTTrY   c                  N   t          d          \  } }}t          | |d           \  }}}}t          |g dd           t          |g dd           t          |g dd           t          |g d           t	          | |d	d
          }t          |dd           t          | |d
          }t          |dd           t          | |d
          }	t          |	dd           t	          | |d          }t          |dd           t          | |d          }t          |dd           t          | |d          }	t          |	dd           t	          | |d          }t          |dd           t          | |d          }t          |dd           t          | |d          }	t          |	dd           t          j        t                    5  t	          | |d           d d d            n# 1 swxY w Y   t          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t          j        t                    5  t          | |dd           d d d            n# 1 swxY w Y   t          | |g dd           \  }}}}t          |g dd           t          |g dd           t          |g dd           t          |g d           d S )NFr   r   )(\?Q?gzG?r2   )HzG?g
ףp=
?rc   )Q?333333?r   )r\   rb   re   r9   r   r   g(\?r   r   gRQ?r   gGz?r   r   r   r   r(  r   )r  g=
ףp=?r  )r  rc   r   )r  r   r  )r\   re   rb   )rX   r'   r   r   r(   r)   r   r   r   r   r    )
rV   rU   rw   rP   r   r   r   r   r   r   s
             rW   )test_precision_recall_f1_score_multiclassr    s}   'u555FFA 1NNNJAq!Qa!3!3!3Q777a!3!3!3Q777a!3!3!3Q777q,,,''' 
1g	F	F	FBb$***	ffg	6	6	6Bb$***	&&'	2	2	2Bb$***		9	9	9Bb$***	ffg	6	6	6Bb$***	&&'	2	2	2Bb$***		<	<	<Bb$***	ffj	9	9	9Bb$***	&&*	5	5	5Bb$***	z	"	" ; ;	::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;	z	"	" 8 8VVY77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	z	"	" 4 433334 4 4 4 4 4 4 4 4 4 4 4 4 4 4	z	"	" A AFFIC@@@@A A A A A A A A A A A A A A A 1yyy$  JAq!Q a!3!3!3Q777a!3!3!3Q777a!3!3!3Q777q,,,'''''sH   G++G/2G/H..H25H2I11I58I5J55J9<J9r   )r   r   r   r   Nc                 $   t          j        g dg          }t          j        g dg          }t          ||g dg |           \  }}}}t          |d           t          |d           t          |d           | t          |g d           d S d S )Nr  r   r   r9   r9   )r   r   r9   r2   )rn   warn_forr   r   r   r9   r9   r   )r>   r   r'   r   )r   rV   rU   rP   r   r   r   s          rW   ;test_precision_refcall_f1_score_multilabel_unordered_labelsr    s     X|||n%%FX|||n%%F0|||b'  JAq!Q q!q!q!1lll+++++ rY   c                  T   t          j        g d          } t          j        g d          }t          | |d           \  }}}}t          | |d          \  }}}}|t          j        |          k    sJ |t          j        |          k    sJ |t          j        |          k    sJ t          | |d          \  }}}}t          j        |           }	|t          j        ||	          k    sJ |t          j        ||	          k    sJ |t          j        ||	          k    sJ d S )N)r   r9   r   r   r9   r9   r   r9   r   r   r9   r   r9   r   r9   )r9   r9   r   r9   r   r9   r9   r9   r9   r   r9   r   r9   r   r9   r   r   r   rz  )r>   r   r'   r   bincountr   )
rV   rU   r   r   r   rw   rP   r   r   r`   s
             rW   .test_precision_recall_f1_score_binary_averagedr    sD   XCCCDDFXCCCDDF 4FFDQQQMBB0QQQJAq!Q0TTTJAq!Qk&!!G
2w///////
2w///////
2w/////////rY   c                     t          j        d          } 	 t          j        g d          }t          j        g d          }t          t	          ||d          dd           t          t          ||d          dd           t          t          ||d          dd           t          j        d	i |  d S # t          j        d	i |  w xY w)
Nraise)rI  )r   r9   r2   r   r9   r2   )r2   r   r9   r9   r2   r   r   r   r~   r2   r   )r>   seterrr   r   r(   r)   r   )old_error_settingsrV   rU   s      rW   test_zero_precision_recallr    s     w///	(,,,--,,,--OFFGLLLcSTUUULIII3PQRRRHVVWEEEsANNN 		''&'''''	''&''''s   BB9 9Cc                  X   t          d          \  } }}t          | |ddg          }t          |ddgddgg           t          | |d	dg          }t          |d
d	gddgg           t          j        |           dz   }t          | |d	|g          }t          |d
dgddgg           d S )NFr   r   r9   r   r   r   r   r2   r$  r\   )rX   r   r   r>   max)rV   rU   rw   r  extra_labels        rW   .test_confusion_matrix_multiclass_subset_labelsr    s    'u555FFA 
&&!Q	8	8	8BrRGaV,--- 
&&!Q	8	8	8BrRGb!W-... &..1$K	&&![1A	B	B	BBrRGaV,-----rY   zlabels, err_msgz,'labels' should contains at least one label.r   r   z.At least one label specified must be in y_truez
empty listzunknown labels)idsc                     t          d          \  }}}t          j        t          |          5  t	          |||            d d d            d S # 1 swxY w Y   d S )NFr   r   r   )rX   r   r   r   r   )rn   r   rV   rU   rw   s        rW   test_confusion_matrix_errorr    s     (u555FFA	z	1	1	1 8 877778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8s   AAArn   r9   r   )NonerK   
multiclassc                     | rt          |           nd}t          j        ||ft                    }t	          g g |           }t          ||           d S )Nr   rN  r   )rq   r>   r   rA   r   r   )rn   expected_n_classesexpectedr  s       rW   *test_confusion_matrix_on_zero_length_inputr    s^     )/5VAx+-?@LLLH	"b	0	0	0Br8$$$$$rY   c            	         g d} t          j        t          |                     }t          | |           }|j        t           j        k    sJ t           j        t           j        t           j        fD ]@}t          | | |	                    |d                    }|j        t           j        k    sJ At           j
        t           j        d t          fD ]@}t          | | |	                    |d                    }|j        t           j        k    sJ At          j        t          |           dt           j                  }t          | | |          }|d         dk    sJ |d         d	k    sJ t          j        t          |           d
t           j                  }t          | | |          }|d         d
k    sJ |d         dk    sJ d S )Nr   F)copyrB  l    r  r   r   r  l    l    )r>   onesrq   r   rN  int64bool_int32uint64astypefloat32float64objectfulluint32)rM   weightr  rN  s       rW   test_confusion_matrix_dtyper  '  s   		AWSVV__F	!Q		B8rx(BHbi0 $ $a&--E-2R2RSSSx28#####*bj$7 & &a&--E-2R2RSSSx2:%%%%% WSVVZry999F	!Qf	5	5	5Bd8z!!!!d8z!!!! WSVV0AAAF	!Qf	5	5	5Bd8*****d8r>>>>>>rY   rN  )Int64Float64booleanc                    t          j        d          }t          j        g d          }|                    ||           }|                    g dd          }t          ||          }t          ||          }t          ||           dS )zkChecks that confusion_matrix works with pandas nullable dtypes.

    Non-regression test for gh-25635.
    pandas)	r9   r   r   r9   r   r9   r9   r   r9   r  )	r   r   r9   r9   r   r9   r9   r9   r9   r  N)r   importorskipr>   r   Seriesr   r   )rN  pd	y_ndarrayrV   y_predictedoutputexpected_outputs          rW   %test_confusion_matrix_pandas_nullabler  B  s     
	X	&	&B44455IYYyY..F))777w)GGKfk22F&y+>>Ov/////rY   c            	          t          j                    } t          | d          \  }}}d}t          ||t	          j        t          | j                            | j                  }||k    sJ d S )NFr[   a|                precision    recall  f1-score   support

      setosa       0.83      0.79      0.81        24
  versicolor       0.33      0.10      0.15        31
   virginica       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
rn   ro   r	   r:   rX   r   r>   r?   rq   ro   rv   rV   rU   rw   rx   ry   s         rW   %test_classification_report_multiclassr  T  s    D'UCCCFFA
O #yT.//00&	  F _$$$$$$rY   c                  J    g dg d}} d}t          | |          }||k    sJ d S )N)	r   r   r   r9   r9   r9   r2   r2   r2   r  a|                precision    recall  f1-score   support

           0       0.33      0.33      0.33         3
           1       0.33      0.33      0.33         3
           2       0.33      0.33      0.33         3

    accuracy                           0.33         9
   macro avg       0.33      0.33      0.33         9
weighted avg       0.33      0.33      0.33         9
r   )rV   rU   rx   ry   s       rW   .test_classification_report_multiclass_balancedr  n  sG    0002M2M2MFF
O #6622F_$$$$$$rY   c                      t          j                    } t          | d          \  }}}d}t          ||          }||k    sJ d S )NFr[   a|                precision    recall  f1-score   support

           0       0.83      0.79      0.81        24
           1       0.33      0.10      0.15        31
           2       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
)r	   r:   rX   r   r  s         rW   :test_classification_report_multiclass_with_label_detectionr    sX    D'UCCCFFA
O #6622F_$$$$$$rY   c            	          t          j                    } t          | d          \  }}}d}t          ||t	          j        t          | j                            | j        d          }||k    sJ d S )NFr[   a|                precision    recall  f1-score   support

      setosa    0.82609   0.79167   0.80851        24
  versicolor    0.33333   0.09677   0.15000        31
   virginica    0.41860   0.90000   0.57143        20

    accuracy                        0.53333        75
   macro avg    0.52601   0.59615   0.50998        75
weighted avg    0.51375   0.53333   0.47310        75
r   )rn   ro   digitsr  r  s         rW   1test_classification_report_multiclass_with_digitsr    s    D'UCCCFFA
O #yT.//00&  F _$$$$$$rY   c                     t          d          \  } }}t          j        g d          |          } t          j        g d          |         }d}t          | |          }||k    sJ d}t          | |g d          }||k    sJ d S )NFr   )bluegreenreda|                precision    recall  f1-score   support

        blue       0.83      0.79      0.81        24
       green       0.33      0.10      0.15        31
         red       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
a|                precision    recall  f1-score   support

           a       0.83      0.79      0.81        24
           b       0.33      0.10      0.15        31
           c       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r   ro   rX   r>   r   r   )rV   rU   rw   rx   ry   s        rW   7test_classification_report_multiclass_with_string_labelr!    s    'u555FFAX...//7FX...//7F
O #6622F_$$$$
O #66PPPF_$$$$$$rY   c                      t          d          \  } }}t          j        g d          }||          } ||         }d}t          | |          }||k    sJ d S )NFr   )u   blue¢u   green¢u   red¢u                precision    recall  f1-score   support

       blue¢       0.83      0.79      0.81        24
      green¢       0.33      0.10      0.15        31
        red¢       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r   rV   rU   rw   rn   rx   ry   s         rW   8test_classification_report_multiclass_with_unicode_labelr$    sp    'u555FFAX:::;;FF^FF^F
O #6622F_$$$$$$rY   c                      t          d          \  } }}t          j        g d          }||          } ||         }d}t          | |          }||k    sJ d S )NFr   )r  greengreengreengreengreenr  a                             precision    recall  f1-score   support

                     blue       0.83      0.79      0.81        24
greengreengreengreengreen       0.33      0.10      0.15        31
                      red       0.42      0.90      0.57        20

                 accuracy                           0.53        75
                macro avg       0.53      0.60      0.51        75
             weighted avg       0.51      0.53      0.47        75
r   r#  s         rW   <test_classification_report_multiclass_with_long_string_labelr'    sp    'u555FFAX22233FF^FF^F
O #6622F_$$$$$$rY   c                      g d} g d}g d}d}t          j        t          |          5  t          | |ddg|           d d d            d S # 1 swxY w Y   d S )	Nr   r   r2   r   r   r   r2   r2   r   r   zclass 0zclass 1zclass 2z6labels size, 2, does not match size of target_names, 3r   r   r2   r  )r   r   r   r   )rV   rU   ro   r   s       rW   =test_classification_report_labels_target_names_unequal_lengthr,    s    __F__F444L
BC	k	-	-	- X XffaV,WWWWX X X X X X X X X X X X X X X X X Xs   AAAc                      g d} g d}g d}d}t          j        t          |          5  t          | ||           d d d            d S # 1 swxY w Y   d S )Nr)  r*  r+  zaNumber of classes, 2, does not match size of target_names, 3. Try specifying the labels parameterr   r  )r   r   r   r   )rV   rU   ro   r   s       rW   @test_classification_report_no_labels_target_names_unequal_lengthr.  	  s    __F__F444L	. 
 
z	1	1	1 I Iff<HHHHI I I I I I I I I I I I I I I I I Is   A

AAc                      d} d}t          d|| d          \  }}t          d|| d          \  }}d}t          ||          }||k    sJ d S )Nr   rt  r9   r   )rO   rN   r  r8   a                precision    recall  f1-score   support

           0       0.50      0.67      0.57        24
           1       0.51      0.74      0.61        27
           2       0.29      0.08      0.12        26
           3       0.52      0.56      0.54        27

   micro avg       0.50      0.51      0.50       104
   macro avg       0.45      0.51      0.46       104
weighted avg       0.45      0.51      0.46       104
 samples avg       0.46      0.42      0.40       104
)r   r   )r  rN   rw   rV   rU   rx   ry   s          rW   %test_multilabel_classification_reportr0    s    II.	YQ  IAv /	YQ  IAvO #6622F_$$$$$$rY   c                  F   t          j        g dg dg          } t          j        g dg dg          }t          | |          dk    sJ t          | |           dk    sJ t          ||          dk    sJ t          |t          j        |                    dk    sJ t          | t          j        |                     dk    sJ t          | t          j        | j                            dk    sJ t          |t          j        | j                            dk    sJ d S )Nr   r   r   r   r   r9   )r>   r   r*   r   r   r=   r   s     rW   $test_multilabel_zero_one_loss_subsetr2  6  s)   	999iii(	)	)B	999iii(	)	)BR  C''''R  A%%%%R  A%%%%R^B//00A5555R^B//00A5555RXbh//00A5555RXbh//00A555555rY   c                  h   t          j        g dg dg          } t          j        g dg dg          }t          j        ddg          }t          | |          dk    sJ t          | |           dk    sJ t          ||          dk    sJ t          |d|z
            dk    sJ t          | d| z
            dk    sJ t          | t          j        | j                            dk    sJ t          |t          j        | j                            d	k    sJ t          | ||
          dk    sJ t          | d|z
  |
          dk    sJ t          | t          j        |           |
          dk    sJ t          | d         |d                   t          | d         |d                   k    sJ d S )Nr   r   r   r9   r   UUUUUU?r   r   r   rB  gUUUUUU?gUUUUUU?)r>   r   r!   r   r=   
zeros_like
sp_hamming)r   r   ws      rW   test_multilabel_hamming_lossr8  D  s   	999iii(	)	)B	999iii(	)	)B
!QAB5((((B1$$$$B1$$$$AF##q((((AF##q((((BHRX..//58888BHRX..//36666Ba000H<<<<AF!444	AAAABM"--Q???7JJJJ1r!u%%BqE2a5)A)AAAAAAArY   c                     t          j        g d          } t          j        g d          }d}t          j        t          |          5  t          | |dd           d d d            n# 1 swxY w Y   t          j        g dg dg          } t          j        g d	g d
g          }d}t          j        t          |          5  t          | |dd           d d d            n# 1 swxY w Y   t          j        g d          } t          j        g d          }d}t          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          | |dd           d d d            d S # 1 swxY w Y   d S )N)r   r9   r   r9   r9   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r   rK   r2   r   r   r   r   r   r   Target is multilabel-indicator but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted', 'samples'\].r   )r   r9   r9   r   r2   r  Target is multiclass but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted'\].r   zJSamplewise metrics are not available outside of multilabel classification.r   zNote that pos_label \(set to 3\) is ignored when average != 'binary' \(got 'micro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   )r>   r   r   r   r   r#   r   r   )rV   rU   r   msg1msg2msg3r   s          rW   test_jaccard_score_validationr@  X  s~   Xooo&&FXooo&&FOG	z	1	1	1 E Effh!DDDDE E E E E E E E E E E E E E E Xyyy))),--FXyyy))),--F	6 	
 
z	.	.	. F Fffh"EEEEF F F F F F F F F F F F F F F Xooo&&FXooo&&F	 	
 
z	.	.	. 8 8ffh77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8WD	z	.	.	. 9 9ffi88889 9 9 9 9 9 9 9 9 9 9 9 9 9 9	  
k	-	-	- D DffgCCCCD D D D D D D D D D D D D D D D D DsZ   
A**A.1A.C&&C*-C*:EE EF  F$'F$G))G-0G-c           	         t          j        g dg dg          }t          j        g dg dg          }t          ||d          dk    sJ t          ||d          dk    sJ t          ||d          dk    sJ t          |t          j        |          d          dk    sJ t          |t          j        |          d          dk    sJ t          |t          j        |j                  d          dk    sJ t          |t          j        |j                  d          dk    sJ t          j        g dg d	g          }t          j        g d
g dg          }t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          t          ||dddg          d           t          t          ||dddg          d           t          t          ||d           t          j        g d                     t          j        g dg dg          }t          j        g d
g dg          }t          t          ||d          d           t          t          ||d          d           d}t          j	        t          |          5  t          ||dgd           d d d            n# 1 swxY w Y   d}t          j	        t          |          5  t          ||dgd           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          t          j        ddgg          t          j        ddgg          d          dk    sJ 	 d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          t          j        ddgddgg          t          j        ddgddgg          d          dk    sJ 	 d d d            n# 1 swxY w Y   t          |           rJ d S )Nr   r   r   r   r   r   r9   r   r   r   r   r   r   r   g?r2   r   r   )r   r   r   r   r   g      ?z	Got 4 > 2r   r   r   z
Got -1 < 0r   zXJaccard is ill-defined and being set to 0.0 in labels with no true or predicted samples.zXJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels.)r>   r   r#   r   r   r=   r   r   r   r   r   r   r/   rM  )recwarnr   r   rV   rU   r>  r?  r   s           rW   test_multilabel_jaccard_scorerC    s   	999iii(	)	)B	999iii(	)	)B
 R333t;;;;R333q8888R333q8888R^B//CCCqHHHHR^B//CCCqHHHHRXbh//CCCqHHHHRXbh//CCCqHHHHXyyy))),--FXyyy))),--FffgFFFPPPffgFFFPPPffiHHH(SSSffiAGGG   ffiAGGG   ffd333RX>U>U>U5V5V   Xyyy))),--FXyyy))),--FffgFFFPPPffjIII7SSSD	z	.	.	. C CffaS'BBBBC C C C C C C C C C C C C C CD	z	.	.	. D DffbT7CCCCD D D D D D D D D D D D D D D	- 
 
,C	8	8	8 
 
"(QF8,,bhAx.@.@'RRR    
 
 
 
 
 
 
 
 
 
 
 
 
 
 
	, 
 
,C	8	8	8 
 
1a&1a&)**1a&1a&)**!  
     
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 G}}sJ   K//K36K3L88L<?L< AN00N47N4A	P..P25P2c           	         g d}g d}g d}t                      }|                    |           |                    |          }|                    |          }t          t          ||          }t          t          ||          }ddgddgddgdgdgdgd g}	ddgdd	gd	dgdgdgd	gd g}
d
D ]>}t          |	|
          D ]+\  }}t           |||           |||                     ,?t          j        ddgddgddgg          }t          j        ddgddgddgg          }t                      5  t	          ||d          dk    sJ 	 d d d            n# 1 swxY w Y   t          |           rJ d S )N)antrE  catrF  rE  rF  birdrG  )rF  rE  rF  rF  rE  rG  rG  rF  )rE  rG  rF  rE  rG  rF  r   r9   r2   )r   r   r   Nr   r   r   )r   rG   	transformr   r#   zipr   r>   r   r   rM  )rB  rV   rU   rn   lbr   r   multi_jaccard_scorebin_jaccard_scoremulti_labels_listbin_labels_listr   m_labelb_labels                 rW   test_multiclass_jaccard_scorerQ    sA   GGGFGGGF###F			BFF6NNNf%%Jf%%J!-@@z:FF						 1v1v1vsQC!dCO 8   #$5 G G 	 	GW##GGDDD!!''BBB   	 X1v1v1v.//FX1v1v1v.//F			 F FVVZ@@@AEEEEEF F F F F F F F F F F F F F F G}}s   3EE Ec                    t          dgdgd          dk    sJ d}t          j        t          |          5  t          ddgddgd          dk    sJ 	 d d d            n# 1 swxY w Y   t          dgdgdd          d	k    sJ t	          j        g d
          }t	          j        g d          }t          t          ||d          d           t          t          ||dd          d           t          |           rJ d S )Nr9   r   rK   r   r~   zOJaccard is ill-defined and being set to 0.0 due to no true or predicted samplesr   r   r   )r9   r   r9   r9   r   )r9   r   r9   r9   r9   r   r:  r   )r#   r   r   r/   r>   r   r   rM  )rB  r   rV   rU   s       rW   !test_average_binary_jaccard_scorerS    s   !qc8444;;;;	'  
,C	8	8	8 F FaVaVX>>>#EEEEEF F F F F F F F F F F F F F F !qcQAAASHHHHXooo&&FXooo&&FffhGGGQQQffh!DDDg   G}}s   A""A&)A&c                  6   t          j        g dg dg          } t          j        g dg dg          }d}t          j        t          |          5  t          | |dd          }|t          j        d          k    sJ 	 d d d            d S # 1 swxY w Y   d S )	Nr   r   r   r   zJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels. Use `zero_division` parameter to control this behavior.r   r   r   r   r   r~   )r>   r   r   r   r/   r#   r   )rV   rU   r   scores       rW   (test_jaccard_score_zero_division_warningrX  	  s     Xyyy))),--FXyyy))),--F	C 
 
,C	8	8	8 + +ffivVVVc*******+ + + + + + + + + + + + + + + + + +s   .BBBzzero_division, expected_scorer  )r9   r   c                 R   t          j        g dg dg          }t          j        g dg dg          }t          j                    5  t          j        dt
                     t          ||d|           }d d d            n# 1 swxY w Y   |t          j        |          k    sJ d S )Nr   rU  rZ  r   rV  )	r>   r   r   r   r\  r/   r#   r   r   )r   expected_scorerV   rU   rW  s        rW   *test_jaccard_score_zero_division_set_valuer[    s     Xyyy))),--FXyyy))),--F		 	"	" 
 
g'=>>>FI]
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 

 FM.11111111s   .BB	Bc                  Z   t          j        g dg dg dg          } t          j        g dg dg dg          }t          | |d           \  }}}}t          |g dd           t          |g dd           t          |g d	d           t          |g d
d           t	          | |dd           }|}t          |g dd           t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          t          j        |                     t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          d|z  |z  d|z  |z   z             t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          t          j        ||                     t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          d           d S )Nr9   r   r   r   r   r9   r   r   r  )r9   r   r9   r   r   )r~   r   r   r~   r2   )r~   r   r   r~   r~   r   r9   r~   )r9   r9   r9   r9   r   r   )r   r  r9   r   r   g      ?r   竪?r   r   r   r   rz  r   r>   r   r'   r   r    r   r   r   rV   rU   rP   r   r   r   f2r`   s           rW   +test_precision_recall_f1_score_multilabel_1re  %  s$   
 X|||\\\<<<@AAFX|||\\\<<<@AAF0NNNJAq!Q a!5!5!5q999a!5!5!5q999a!7!7!7;;;aq111	VV!T	:	:	:BGb///1555 1QQQJAq!Q7###3+,,,999FFG<<<bgbkk  
 1QQQJAq!Q333999FFG<<<	!a1q519%   1TTTJAq!Q7###3+,,,999FFJ???

2w'''   1SSSJAq!Q333999FFINNNPSTTTTTrY   c                  \   t          j        g dg dg dg          } t          j        g dg dg dg          }t          | |d           \  }}}}t          |g dd           t          |g d	d           t          |g d
d           t          |g dd           t	          | |dd           }|}t          |g dd           t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          d|z  |z  d|z  |z   z             t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          t          j        |                     t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          t          j        ||                     t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          dd           d S )Nr]  r^  r  r   r   r   r9   r  r   )r~   r   r~   r~   r2   )r~   r   r~   r~   )r~   gQ?r~   r~   r9   r2   r9   r   r`  )r   皙?r   r   r         ?r   r   r   g      ?r4  r   r   ra   rz  r   g&S?rb  rc  s           rW   +test_precision_recall_f1_score_multilabel_2rk  h  s/    X|||\\\<<<@AAFX|||\\\<<<@AAF 1NNNJAq!Qa!5!5!5q999a!5!5!5q999a!6!6!6:::aq111	VV!T	:	:	:BGb///15550QQQJAq!Q4   4   0111999FFG<<<	!a1q519%  
 1QQQJAq!Q4   5!!!6"""999FFG<<<bgbkk   1TTTJAq!Q5!!!5!!!=)))999FFJ???

2w'''  
 1SSSJAq!Q 5!!!5!!!=)))999FFI>>>    rY   c           	         t          j        g dg dg dg          }t          j        g dg dg dg          }| dk    rdnd} t          ||d |           \  }}}}t          || dddgd	           t          |dd
d| gd	           t          |g dd	           t          |g dd	           t	          ||d	d |           }|}t          |g dd	           t          ||d|           \  }}}}t          |d	| z   dz             t          |d| z   dz             t          |d           |J t          t	          ||d	d          t          j        |                     t          ||d|           \  }}}}t          |d           t          |d
           t          |d           |J t          t	          ||d	d|           d|z  |z  d|z  |z   z             t          ||d|           \  }}}}t          || dk    rdnd           t          |d
           t          |d           |J t          t	          ||d	d|           t          j        ||                     t          ||d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          ||d	d|           d d	           d S )!Nr^  r]  r  r  rg  r   r~   rV  r2   r   r_  rh  r   r   r   )r   ri  r9   r   r   r         ?ra  r`  r   r   rd   r   r   r   r   g?rz  r   r   ra   gZd;O?rb  )	r   rV   rU   rP   r   r   r   rd  r`   s	            rW   7test_precision_recall_f1_score_with_an_empty_predictionro    s    X|||\\\<<<@AAFX|||\\\<<<@AAF
 )C//CCSM0M  JAq!Q a-c3!?CCCa#sC!?CCCa!7!7!7;;;aq111	VV!T	W	W	WBGb///15550}  JAq!Q A-2333C-/14555?+++999FFG<<<bgbkk   1}  JAq!Q 5!!!30111999FG=	
 	
 	
 
!a1q519%	   1
-  JAq!Q MQ$6$655C@@@3,---999FJm	
 	
 	
 	
2w'''	   1SSSJAq!Q 5!!!5!!!5!!!999FI]	
 	
 	
 		    rY   r   )r   r   r   r   c                    t          j        d          }t          j        |          }t          t          |||| |          \  }}}}t          t
          ||| ||          }	t          |          }t          ||           t          ||           t          ||           |J t          |	t          |                     d S )Nre   r   r   r   r   rm  )r>   r   r5  r   r'   r    rt   r   )
r   r   r   rV   rU   rP   r   r   r   fbetas
             rW   "test_precision_recall_f1_no_labelsrt    s     XgF]6""F#'#  JAq!Q #  E -((M=)))=)))=)))999u]3344444rY   c                    t          j        d          }t          j        |          }t          }t	          j        t                    5   |||| d          \  }}}}d d d            n# 1 swxY w Y   t          |d           t          |d           t          |d           |J t	          j        t                    5  t          ||| d          }d d d            n# 1 swxY w Y   t          |d           d S )Nrq  r   r  r   )	r>   r   r5  r'   r   r   r/   r   r    )	r   rV   rU   funcrP   r   r   r   rs  s	            rW   1test_precision_recall_f1_no_labels_check_warningsrw    s   XgF]6""F*D	,	-	- E ET&&'DDD
1aE E E E E E E E E E E E E E E 111999	,	-	- G GFFG#FFFG G G G G G G G G G G G G G G q!!!!!s$   	A**A.1A.C""C&)C&c                    t          j        d          }t          j        |          }t          t          ||d d|           \  }}}}t          t
          ||dd |           }t          |           } t          || | | gd           t          || | | gd           t          || | | gd           t          |g dd           t          || | | gd           d S )Nrq  r   rr  rm  r2   rU  )r>   r   r5  r   r'   r    rt   r   )r   rV   rU   rP   r   r   r   rs  s           rW   /test_precision_recall_f1_no_labels_average_nonery  ,  s   XgF]6""F $'#  JAq!Q VV#t=  E -((Ma-!NPQRRRa-!NPQRRRa-!NPQRRRaA...em]M%RTUVVVVVrY   c                  .   t          j        d          } t          j        |           }t          j        t
                    5  t          | |d d          \  }}}}d d d            n# 1 swxY w Y   t          |g dd           t          |g dd           t          |g dd           t          |g dd           t          j        t
                    5  t          | |dd           }d d d            n# 1 swxY w Y   t          |g dd           d S )Nrq  r9   r  rU  r2   r`  )	r>   r   r5  r   r   r/   r'   r   r    )rV   rU   rP   r   r   r   rs  s          rW   4test_precision_recall_f1_no_labels_average_none_warnr{  N  s   XgF]6""F 
,	-	- 
 
4FDq
 
 

1a
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 aA...aA...aA...aA...	,	-	- B BFFDAAAB B B B B B B B B B B B B B B eYYY22222s$   A''A+.A+C77C;>C;c            	         t           t          }} dD ]}d}t          j        ||          5   | g dg d|           d d d            n# 1 swxY w Y   d}t          j        ||          5   | g dg d|           d d d            n# 1 swxY w Y   d}t          j        ||          5   | t	          j        d	d
gd	d
gg          t	          j        d	d
gd
d
gg          d           d d d            n# 1 swxY w Y   d}t          j        ||          5   | t	          j        d	d
gd
d
gg          t	          j        d	d
gd	d
gg          d           d d d            n# 1 swxY w Y   d}t          j        ||          5   | t	          j        d	d	gd	d	gg          t	          j        d
d
gd
d
gg          d           d d d            n# 1 swxY w Y   d}t          j        ||          5   | t	          j        d
d
gd
d
gg          t	          j        d	d	gd	d	gg          d           d d d            n# 1 swxY w Y   d}t          j        ||          5   | d	d	gddgd           d d d            n# 1 swxY w Y   d}t          j        ||          5   | ddgd	d	gd           d d d            n# 1 swxY w Y   t          j        d          5 }t          j        d           t          d
d
gd
d
gd           d}t          |
                                j                  |k    sJ d}t          |
                                j                  |k    sJ 	 d d d            d S # 1 swxY w Y   d S )NNr   r   zPrecision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.r   r   r9   r9   r2   r   zRecall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.zPrecision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.r9   r   r   zRecall and F-score are ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.zPrecision and F-score are ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.r   zRecall and F-score are ill-defined and being set to 0.0 due to no true samples. Use `zero_division` parameter to control this behavior.r   rK   Tr   always)r'   r/   r   r   r>   r   r   r   r\  r   popr   )r   r7  r   r   r   s        rW   test_prf_warningsr  j  s   *,BqA. 5 5 	 \!3''' 	5 	5AiiiG4444	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	 \!3''' 	5 	5AiiiG4444	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5
	  
as	#	#	# U U	"(QFQF#
$
$bhAA/?&@&@)TTTTU U U U U U U U U U U U U U U	  
as	#	#	# U U	"(QFQF#
$
$bhAA/?&@&@)TTTTU U U U U U U U U U U U U U U
	  
as	#	#	# S S	"(QFQF#
$
$bhAA/?&@&@'RRRRS S S S S S S S S S S S S S S	  
as	#	#	# S S	"(QFQF#
$
$bhAA/?&@&@'RRRRS S S S S S S S S S S S S S S
	  
as	#	#	# . .	1a&2r(H----. . . . . . . . . . . . . . .	  
as	#	#	# . .	2r(QFH----. . . . . . . . . . . . . . . 
	 	-	-	- 0h''''AAIIII 	 6::<<'((C//// 	 6::<<'((C/////!0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   A

A	A	-BB	B	0?C;;C?C??E))E-0E-?GGG:?II	I	(JJJ*K		KK)BM>>NNc           	         t           }dD ]2}t          |g dg d||            t          |g dg d||            3t          |t          j        ddgddgg          t          j        ddgddgg          d|            t          |t          j        ddgddgg          t          j        ddgddgg          d|            t          |t          j        ddgddgg          t          j        ddgddgg          d|            t          |t          j        ddgddgg          t          j        ddgddgg          d|            t          |ddgd	d	gd
|            t          |d	d	gddgd
|            t	          j        d          5 }t	          j        d           t          ddgddgd
|            t          |          dk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr}  r   r~  rV  r9   r   r   r   r   rK   Tr   r  )r'   r   r>   r   r   r   r\  rq   )r   r   r   r   s       rW   )test_prf_no_warnings_if_zero_division_setr    s    	(A. 
 
yyy)))WM	
 	
 	
 	
 	yyy)))WM	
 	
 	
 	
 	

 	
1a&1a&!""
1a&1a&!""#    	
1a&1a&!""
1a&1a&!""#    	
1a&1a&!""
1a&1a&!""#    	
1a&1a&!""
1a&1a&!""#    	Aq6B8X]    	B8aVX]    
	 	-	-	-  h''''FQFHM	
 	
 	
 	
 6{{a                                   s   AG!!G%(G%c           	         t          t          t          j        ddgddgg          t          j        ddgddgg          d|            t	          j        d          5 }t	          j        d           t          t          j        ddgddgg          t          j        ddgddgg          d|            | dk    r-t          |                                j	                  d	k    sJ nt          |          dk    sJ t          ddgddg           | dk    r,t          |                                j	                  d	k    sJ d d d            d S # 1 swxY w Y   d S )
Nr9   r   r   rV  Tr   r  r   zzRecall is ill-defined and being set to 0.0 due to no true samples. Use `zero_division` parameter to control this behavior.)r   r)   r>   r   r   r   r\  r   r  r   rq   r   r   s     rW   test_recall_warningsr    s   
1a&1a&!""
1a&1a&!""#    
	 	-	-	- h'''Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 F""FJJLL())"" " " " " v;;!####aVaV$$$F""FJJLL())"" " " "-                 s   C&EEEc           	         t          j        d          5 }t          j        d           t          t	          j        ddgddgg          t	          j        ddgddgg          d|            | dk    r-t          |                                j                  d	k    sJ nt          |          dk    sJ t          ddgddg           | dk    r,t          |                                j                  d	k    sJ d d d            n# 1 swxY w Y   t          t          t	          j        ddgddgg          t	          j        ddgddgg          d|            d S )
NTr   r  r9   r   r   rV  r   zPrecision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.)r   r   r\  r(   r>   r   r   r  r   rq   r   r  s     rW   test_precision_warningsr  5  s   		 	-	-	- h'''Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 F""FJJLL())"" " " " " v;;!####AA'''F""FJJLL())"" " " "-              : 
1a&1a&!""
1a&1a&!""#     s   C&DDDc           
      0   t          j        d          5 }t          j        d           t          t	          t
          d          fD ]3} |t          j        ddgddgg          t          j        ddgddgg          d| 	           t          |          dk    sJ  |t          j        ddgddgg          t          j        ddgddgg          d| 	           t          |          dk    sJ  |t          j        ddgddgg          t          j        ddgddgg          d| 	           | d
k    r.t          |
                                j                  dk    sJ t          |          dk    sJ 5	 d d d            d S # 1 swxY w Y   d S )NTr   r  r2   r   r9   r   r   rV  r   zF-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.)r   r   r\  r   r   r    r>   r   rq   r   r  r   )r   r   rW  s      rW   test_fscore_warningsr  ]  s7   		 	-	-	- #(h'''! < < <=  	(  	(EE1a&1a&)**1a&1a&)**+	    v;;!####E1a&1a&)**1a&1a&)**+	    v;;!####E1a&1a&)**1a&1a&)**+	    &&

,---- - - - - 6{{a'''''A 	(#( #( #( #( #( #( #( #( #( #( #( #( #( #( #( #( #( #(s   E'FFFc                     g d} g d}d}t          j        g dg dg dg          }t          j        g dg dg dg          }d}| ||f|||ffD ]q\  }}}t          t          t          t          t          d	
          fD ]A}	t          j        t          |          5   |	||           d d d            n# 1 swxY w Y   Brd S )N)r9   r2   r   r   )r9   r2   r   r9   r<  r   r   r   r1  r;  r2   r   r   )
r>   r   r(   r)   r   r   r    r   r   r   )
	y_true_mc	y_pred_mcmsg_mc
y_true_ind
y_pred_indmsg_indrV   rU   r   r{   s
             rW   'test_prf_average_binary_data_non_binaryr    sV   II	1 
 999iii;<<J999iii;<<J	<  
Iv&	Z)  ' '
 Ka(((	
 	' 	'F z555 ' 'vv&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' '	'	' 's   B66B:=B:c                     d} d}d}d}d}d}| t          j        g dg dg d	g          f| t          j        d
dgdd
gddgg          f|g df|g df|g df|t          j        dgdgdgg          f|t          j        d
gdgdgg          f|t          j        dgdgdgg          f|t          j        d
dgddgddgg          f|t          j        ddgddgddgg          fg
}i | | f| ||f|||f||| fd || fd ||f|||fd ||fd ||fd | |fd ||fd ||fd ||fd ||fd | |fd ||fd ||fd ||fd | |fd ||fd ||fd i}t          |d          D ]\  \  }}	\  }
}	 |||
f         }n# t          $ r ||
|f         }Y nw xY w|t	          j        t                    5  t          |	|           d d d            n# 1 swxY w Y   ||
k    rZd                    ||
          }t	          j        t          |          5  t          |	|           d d d            n# 1 swxY w Y   |||| fvrXd                    |          }t	          j        t          |          5  t          |	|           d d d            n# 1 swxY w Y   7t          |	|          \  }}}||k    sJ |	                    d          r|j        dk    sJ |j        dk    sJ nDt          |t          j        |	                     t          |t          j        |                     t	          j        t                    5  t          |	d d         |           d d d            n# 1 swxY w Y   ddg}	d d!g}d"}t	          j        t          |          5  t          |	|           d d d            d S # 1 swxY w Y   d S )#Nzmultilabel-indicatorr  rK   
continuouszmulticlass-multioutputzcontinuous-multioutputr   r   r   r   r9   )r2   r   r9   )r~   rn  r   r2   r   r~   rn  r   r   rY  g?g      @)r  z@Classification metrics can't handle a mix of {0} and {1} targetsr   z{0} is not supported
multilabelcsrr   )r9   r2   )r   r2   r   )r2   )r   r2   zYou appear to be using a legacy multi-label data representation. Sequence of sequences are no longer supported; use a binary array or sparse matrix instead - the MultiLabelBinarizer transformer can convert to this format.)r>   r   r   KeyErrorr   r   r   r.   format
startswithr   squeeze)INDMCBINCNTMMCMCNEXAMPLESEXPECTEDtype1r   type2r   r  r   merged_typey1outy2outr   s                     rW   test__check_targetsr    sj    !C	B
C
C
"C
"C 
bh			999iii899:	bhAAA/001	YYY	iii	ooo	RXsQC!o&&'	bhaS1#''(	bhuse,--.	bhAAA/001	bhc
S#Jc
;<<=H	c
C	R" 
c
C 
S	4	
 
c
D 
b	2 
c
D 
c
D 
c
D 
c
D 
S	4 
c
D 
c
D 
c
D  
c
D!" 
S	4#$ 
c
D%& 
c
D	c
D	S	4	c
D- H2 %,HQ$?$?$? !, !, [eR	.u-HH 	. 	. 	.u-HHH	.z** ' 'r2&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ~~--3VE5-A-A  ]:W=== + +"2r***+ + + + + + + + + + + + + + + b#..4;;EBBGzAAA / /&r2.../ / / / / / / / / / / / / / / )7r2(>(>%K(****%%l33 :|u,,,,|u,,,,,"5"*R..999"5"*R..999z** , ,r#2#w+++, , , , , , , , , , , , , , , )	B
B	3  
z	-	-	-  r2                 sl   .
E99FF/GG	G	H++H/	2H/	.JJ	J	M&&M*	-M*	N66N:=N:c                  N    ddg} ddg}t          | |          d         dk    sJ d S )Nr   r9   r   r  r-   ra  s     rW   Atest__check_targets_multiclass_with_both_y_true_and_y_pred_binaryr  	  s:    VFWF&&))!,<<<<<<rY   c                     t          j        g d          } t          j        g d          }t          | |          dk    sJ t          j        g d          } t          j        g d          }t          | |          dk    sJ d S )N)r   r9   r9   r   )g      !r   rn  g333333ӿ333333?)r   r2   r2   r   )r>   r   r"   rV   pred_decisions     rW   test_hinge_loss_binaryr  	  s    Xnnn%%FH33344Mfm,,7777Xlll##FH33344Mfm,,777777rY   c            
         t          j        g dg dg dg dg dg dg          } t          j        g d          }t          j        d| d         d         z
  | d         d         z   d| d         d         z
  | d         d	         z   d| d	         d	         z
  | d	         d
         z   d| d
         d         z
  | d
         d	         z   d| d         d
         z
  | d         d	         z   d| d         d	         z
  | d         d
         z   g          }t          j        |dd |           t          j        |          }t	          ||           |k    sJ d S )N
ףp=
?(\ſ(\gGz)HzGgGz׿Q޿r  333333r  RQؿr  )r  r  r  r  gzGgHzGgHzGѿgQ?)r   r9   r2   r9   r   r2   r9   r   r2   r   r   r   outr>   r   clipr   r"   )r  rV   dummy_lossesdummy_hinge_losss       rW   test_hinge_loss_multiclassr  	  s   H((((((((((((((((((	
	 	M X((())F8a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	 	L GL!T|4444w|,,fm,,0@@@@@@@rY   c                      t          j        g d          } t          j        g dg dg dg dg          }d}t          j        t          |          5  t          | |           d d d            d S # 1 swxY w Y   d S )N)r   r9   r2   r2   )gRQ?g rh?g(\gffffffr  r  zDPlease include all labels in y_true or pass labels as third argumentr   )r>   r   r   r   r   r"   )rV   r  error_messages      rW   :test_hinge_loss_multiclass_missing_labels_with_labels_noner  2	  s    Xlll##FH((((((((((((		
 M 	O  
z	7	7	7 * *6=)))* * * * * * * * * * * * * * * * * *s   A22A69A6c            
         t          j        g d          } t          j        g d          }d}t          j        t          t          j        |                    5  t          | |           d d d            n# 1 swxY w Y   t          j        ddgddgddgddgddgddgddgg          }g d	}d
}t          j        t          t          j        |                    5  t          | ||           d d d            d S # 1 swxY w Y   d S )N)r2   r9   r   r9   r   r9   r9   )r   r9   r2   r9   r   r2   r9   zThe shape of pred_decision cannot be 1d arraywith a multiclass target. pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7,)r   r  r   r9   r2   r   zThe shape of pred_decision is not consistent with the number of classes. With a multiclass target, pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7, 2))rV   r  rn   )r>   r   r   r   r   reescaper"   )rV   r  r  rn   s       rW   <test_hinge_loss_multiclass_no_consistent_pred_decision_shaper  C	  s    X+++,,FH22233M	  
z=)A)A	B	B	B ? ?&>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ? Hq!fq!fq!fq!fq!fq!fqRSfUVVMYYF	  
z=)A)A	B	B	B N N&fMMMMN N N N N N N N N N N N N N N N N Ns$   A::A>A>!DDDc            	         t          j        g dg dg dg dg dg          } t          j        g d          }t          j        g d          }t          j        d| d         d         z
  | d         d         z   d| d         d         z
  | d         d         z   d| d         d         z
  | d         d	         z   d| d	         d         z
  | d	         d         z   d| d
         d         z
  | d
         d	         z   g          }t          j        |dd |           t          j        |          }t	          || |          |k    sJ d S )Nr  皙r  r  r  r  )r   r9   r2   r9   r2   )r   r9   r2   r   r9   r   r2   r   r   r  r   r  r  rV   rn   r  r  s        rW   .test_hinge_loss_multiclass_with_missing_labelsr  _	  sz   H(((((((((((((((	
 M Xooo&&FXlll##F8a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
 L GL!T|4444w|,,fmF;;;?OOOOOOOrY   c            	         t          j        g dg dg dg dg dg          } t          j        g d          }t          j        g d          }t          j        d| d         d         z
  | d         d         z   d| d         d	         z
  | d         d         z   d| d	         d	         z
  | d	         d         z   d| d
         d         z
  | d
         d	         z   d| d         d	         z
  | d         d         z   g          }t          j        |dd |           t          j        |          }t	          t          || |          |           d S )N)r  r  r  )g333333ÿr  r  )r  r  r  )r  g(\gzGڿ)r   r2   r2   r   r2   r   r9   r   r2   r   r   r  r   )r>   r   r  r   r   r"   r  s        rW   @test_hinge_loss_multiclass_missing_labels_only_two_unq_in_y_truer  y	  s   
 H!!!!!!!!!!!!!!!	
 M Xooo&&FXiii  F8a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
 L GL!T|4444w|,,6=888:J    rY   c            
      V   g d} g dg dg dg dg dg dg}t          j        d|d         d         z
  |d         d         z   d|d         d         z
  |d         d         z   d|d         d         z
  |d         d	         z   d|d	         d         z
  |d	         d         z   d|d
         d	         z
  |d
         d         z   d|d         d         z
  |d         d	         z   g          }t          j        |dd |           t          j        |          }t	          | |          |k    sJ d S )N)r  r  r  r  whiter  r  r  r  r  r9   r   r2   r   r   r   r  r  )rV   r  r  r  s       rW   +test_hinge_loss_multiclass_invariance_listsr  	  sw    ?>>F$$$$$$$$$$$$$$$$$$M 8a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	 	L GL!T|4444w|,,fm,,0@@@@@@@rY   c                  @   g d} t          j        ddgddgddgddgddgd	d
gg          }t          | |          }t          j        t	          j        t          j        |           dk    |d d df                              }t          ||           g d} g dg dg dg}t          | |d          }t          |d           | dz  } |dz  }t          | |d          }t          |dd           t          j        |          dk    }t          | |dd          }t          |t          | t          j        |dd                               t          ddgddgd          dk    sJ t          ddgddgd          t           j	        k    sJ t          ddgddgd          t           j	        k    sJ t          g dg dg dg dgd          dk    sJ t          g dg d g dg dgd          t           j	        k    sJ g d} d!d"gd#dgd$dgg}t          j        t                    5  t          | |           d d d            n# 1 swxY w Y   g d%} d!d"gd#dgd$dgd"d!gg}t          | |          }t          |d&d           ddg} d!d"gd#dgg}t          j        ddgddgg          }d'}t          j        t          |(          5  t          | |           d d d            n# 1 swxY w Y   d!d"gd#dgd!d)gg}d*}t          |t          | |f t          j        t          j        |d d df                              }t          | |ddg+          }t          ||           g d,} g d-g d.g d/g}t          | |g d0+          }t          |d1d           d S )2N)nor  r  yesr  r  r   r   rc   g{Gz?gGz?r   rj  gMbP?g+?r  r9   rD  )r  ffffff?r   )r   r  r  )r   r   r  TrK  g躕ʀ?r2   Fg.L`@r   r   )rL  epsr   r  r   r   r1  r   )r   r   r   r  r  r   r   hamspamr  r  L7?zly_true contains only one label \(2\). Please provide the true labels explicitly through the labels argument.r   r  zBFound input variables with inconsistent numbers of samples: [3, 2]r   r   )r  r  r  )r   r   r  )r  rc   r   rC  gPv0?)r>   r   r$   r   r   logpmfr   asarrayr  r   r   r   r   log)	rV   rU   loss	loss_truer   	error_strtrue_log_losscalculated_log_lossy_score2s	            rW   test_log_lossr  	  s   444FX
sc3Z$SzD$<%QVX F FF##D)"(6*:*:e*CVAAAqD\RRSSSIi((( YYFoo@FFFd333Di((( aKF
aKFFFe444DmQ7777 Z#%FFFd<<<DhvrwvsC/H/HIIJJJ QFQF***a////QFQF***bf4444QFQF***bf4444 III			999iii@aHHHAMMMMIIIyyy)))D!LLLPRPVVVVV YYFCj3*sCj1F	z	"	" ! !   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ,++FCj3*sCj3*=FFF##Di3333 VFCj3*%Fhc
S#J/00G	C  
z	3	3	3 ! !   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! Cj3*sCj1FTIHff55 WRVGAAAqDM22333M"67Aq6BBB+];;; YYF///BHFHYYY777Di333333s$   /IIIK++K/2K/c                     t          j        ddg|           }|                                }t          ||d          }t          j        |          sJ dS )zCheck the behaviour of `eps="auto"` that changes depending on the input
    array dtype.
    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/24315
    r   r9   r  autor  N)r>   r   r  r$   isfinite)global_dtyperV   rU   r  s       rW   test_log_loss_eps_autor  
  sY     Xq!fL111F[[]]FFF///D;trY   c                      t          j        ddgt           j                  } |                                 }t	          | |d          }t          j        |          sJ dS )z2Check the behaviour of `eps="auto"` for np.float16r   r9   r  r  r  N)r>   r   float16r  r$   r  )rV   rU   r  s      rW   test_log_loss_eps_auto_float16r  
  sY    Xq!fBJ///F[[]]FFF///D;trY   c                  r   t          j        g d          } t          j        ddgddgddgddgg          }t          t          fg}	 dd	lm}m} |                    ||f           n# t          $ r Y nw xY w|D ]=\  }} ||            ||          }}t          ||          }	t          |	d
d           >d S )Nr  r  r  r   r   r   r   r   )r  	DataFramer  r   r   )
r>   r   r   r  r  r  r}  ImportErrorr$   r   )
y_try_prtypesr  r  TrueInputTypePredInputTyperV   rU   r  s
             rW   test_log_loss_pandas_inputr  
  s   822233D8c3Z#sc3Z#sDEED]+,E,,,,,,,,fi())))   (- 8 8$}&t,,mmD.A.A''D)Q77777	8 8s   A' '
A43A4c                     t          j        g d          } t          j        g d          }t          j        | |z
            dz  t	          |           z  }t          t          | |           d           t          t          | |          |           t          t          d| z   |          |           t          t          d| z  dz
  |          |           t          j        t                    5  t          | |dd                     d d d            n# 1 swxY w Y   t          j        t                    5  t          | |dz              d d d            n# 1 swxY w Y   t          j        t                    5  t          | |dz
             d d d            n# 1 swxY w Y   t          j        g d          } t          j        g d          }d	}t          j        t          |
          5  t          | |           d d d            n# 1 swxY w Y   t          t          dgdg          d           t          t          dgdg          d           t          t          dgdg          d           t          t          dgdgd          d           t          t          dgdgd          d           d S )N)r   r9   r9   r   r9   r9   )r   r   rc   r  r   gffffff?r2   r~   r   r9   )r   r9   r2   r   )r   r   r   r  zMOnly binary classification is supported. The type of the target is multiclassr   r   r   g{Gz?r   r  foobar)r   )
r>   r   r   normrq   r   r+   r   r   r   )rV   rU   
true_scorer  s       rW   test_brier_score_lossr  +
  s   X((())FX55566FVf_--2S[[@J(88#>>>(88*EEE(vv>>
KKK(Va@@*MMM	z	"	" - -,,,- - - - - - - - - - - - - - -	z	"	" / /#.../ / / / / / / / / / / / / / /	z	"	" / /#.../ / / / / / / / / / / / / / / Xlll##FX***++FW  
z	7	7	7 ) )((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) ("u55t<<<(!se44d;;;(!se44d;;;(%3%5III4PPP(%3%5III4PPPPPsH   1DDD:EE!E>FF"%F"2HHHc                      d} t          j        t          |           5  t          g dg d           d d d            d S # 1 swxY w Y   d S )Nz%y_pred contains classes not in y_truer   rU  r   )r   r   r   r   r   s    rW   #test_balanced_accuracy_score_unseenr  N
  s    
1C	k	-	-	- 6 6			9995556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s   A  AAzy_true,y_pred)r   r   r   r   )r   r   r   r   )r   r   r   r   c                    t          | |dt          j        |                     }t                      5  t	          | |          }d d d            n# 1 swxY w Y   |t          j        |          k    sJ t	          | |d          }t	          | t          j        | | d                             }|||z
  d|z
  z  k    sJ d S )Nr   r   T)adjustedr   r9   )r)   r>   uniquer   r   r   r   	full_like)rV   rU   macro_recallbalancedr  chances         rW   test_balanced_accuracy_scorer  T
  s     	&0A0A  L 
		 ; ;*66::; ; ; ; ; ; ; ; ; ; ; ; ; ; ; v}\222222&vvEEEH$VR\&&)-L-LMMF6)a&j9999999s   AAA)NF)	functoolsr   	itertoolsr   r   r   r   r  numpyr>   scipyr   scipy.statsr   r   sklearnr	   r
   sklearn.datasetsr   sklearn.preprocessingr   r   sklearn.utils.validationr   sklearn.utils._testingr   r   r   r   r   r   sklearn.utils._mockingr   sklearn.metricsr   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r&   r'   r(   r)   r*   r+   r,   sklearn.metrics._classificationr.   sklearn.exceptionsr/   scipy.spatial.distancer0   r6  rX   r|   r   markparametrizer   r   r   r   r   r   r   r   r   r   r   r  r  r-  r@  rE  rS  rW  r`  r   rf  rj  rp  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r!  r$  r'  r,  r.  r0  r2  r8  r@  rC  rQ  rS  rX  r[  re  rk  ro  rt  rw  ry  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   rY   rW   <module>r     s:                     " " " " " "  				           ! ! ! ! ! !              ; ; ; ; ; ; @ @ @ @ @ @ @ @ 7 7 7 7 7 7 6 6 6 6 6 6 5 5 5 5 5 5 < < < < < < 2 2 2 2 2 2 5 5 5 5 5 5 2 2 2 2 2 2 0 0 0 0 0 0 * * * * * * 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 1 1 1 1 1 1 - - - - - - , , , , , , $ $ $ $ $ $ ' ' ' ' ' ' ( ( ( ( ( ( & & & & & & ) ) ) ) ) ) $ $ $ $ $ $ - - - - - - ; ; ; ; ; ; + + + + + + ( ( ( ( ( ( ) ) ) ) ) ) , , , , , , 7 7 7 7 7 7 : : : : : : 5 5 5 5 5 5 8 8 8 8 8 8)' )' )' )'`?@ ?@ ?@DW W W< ...99  :97 7 7
 
 
D   $ *J *J *JZ M M M,
0 
0 
09 9 9
; 
; 
; N N N 
 
 
 = = =*	= 	= 	= O  O  OF&S &S &SRT T T< +   % % %9 9 9; ; ;& 
 #"(#5#5#566""(#5#5#566  A	
 #"(#5#5#566""(#5#5#566  I	
 #"(#5#5#566""(#5#5#566  :	
 #"(#5#5#566""(#5#5#566  I	
 #"(#5#5#566""(#5#5#566  O	
E), ,Z* *[, ,Z*  #"(???33""(???33 8	
	 * * *" " "2  >4 4 4
  $2 $2 $2N R  R  RF5 5 5p c5\22U U 32UB7( 7( 7(t $S$S$STT, , UT,0 0 0"( ( (". . .& 	;<
QAB 
'(   8 8 8 taVYYY'-M-M-M   % % %  6 "A"A"ABB0 0 CB0"% % %4% % %$% % %(% % %6 %  %  %F% % %,% % %.X X XI I I % % %<6 6 6B B B(%D %D %DPJ J JZ" " "J  .+ + + 868:LMM	2 	2 NM	2 ?U ?U ?UD = = =@ ...99I I :9 IX !%%$M$M$MNN1a&115 5 21 ON &%5< $M$M$MNN" " ON"& 1a&11W W 21WB3 3 38`0 `0 `0F 1a&11=  =  21= @ ...99# # :9#L ...99$ $ :9$N ...99$( $( :9$(N' ' '>_ _ _D= = =8 8 8A A A6* * *"N N N8P P P4  @A A A8K4 K4 K4\
 
 
  8 8 8$ Q  Q  QF6 6 6 			3334			3334			3334 
: 
: 
: 
: 
:rY   