
     h'2                       d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dl	mZ d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlm Z  d dlm!Z! d dlm"Z" d dlm#Z# d dlm$Z$ d dlm%Z% d dl&m'Z'm(Z( d dlm)Z)m*Z* d dlm+Z+ d dl,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3 e e"e%gZ4dd!Z5d" Z6d# Z7d$ Z8d% Z9ej:        ;                    d&d'd g          d(             Z<d) Z=d* Z>d+ Z?d, Z@d- ZAd. ZBd/ ZCd0 ZDd1 ZEd2 ZFd3 ZGej:        ;                    d4 ejH        g d5          g d6f ejH        g d5          dfg d7g d8fg d7dfg          d9             ZIej:        ;                    d4 ejH        g d:          g d6f ejH        g d;          g d<fg          d=             ZJej:        ;                    d4 ejH        g d>          dfg d?dfg d>g d6fg d?g d8fg          d@             ZKej:        ;                    dAg dB          dC             ZLdD ZMej:        ;                    dEdF ejH        g d>          g dGfdF ejH        g d?          g dHfdI ejH        g d:          dfdJ ejH        g d?          g dKfdL ejH        g d>          d dMgfdL ejH        g d?          dNdOgfdP ejH        g d>          g dQfdP ejH        g d?          g dRfdS ejH        g dT          g d8fdS ejH        g dR          g d8fdS ejH        g dQ          g d6fg          ej:        ;                    dUdVdWg          dX                         ZNej:        ;                    dYdZd[dVd\fd]d[dWd\fd^dVg d_fd`dVdadbfdcdUddifdei fg          df             ZOdg ZPej:        ;                    dhe4          di             ZQej:        ;                    dhe4          dj             ZRej:        ;                    dhe4          dk             ZSdl ZTdm ZUdn ZVdo ZWdp ZXdq ZYej:        ;                    drg dsg dtd gd gfg dsg dud gd gfg dsg dvd gd gfg dsg dwd gd gfg dxg dtdagd gfg dxg dudagd gfg dxg dvdagd gfg dxg dwdagd gfg dyg dtdzgd gfg dyg dud gd gfg dyg dvd gd gfg dyg dwd gd gfg d{g dtg d|g dyfg d{g dug d|g dyfg d{g dvg d|g dyfg d{g dwg d|g dyfg d}g dtg d~g dfg d}g dug d~g dfg d}g dvg d~g dfg d}g dwg d~g dfg          d             ZZej:        ;                    drdMd gdadagdMgd gfd dMgdadagdMgd gfg dsg ddagd gfg dxg ddagd gfg dyg dd gd gfg d{g ddMgd gfg d}g ddMgd gfg d~g ddMgd gfg          d             Z[d Z\ej:        ;                    dg d          d             Z]ej:        ;                    dg dg dg dg dg dg          d             Z^ej:        ;                    dd dMgg dtdfg dyd dagdfg dg dtdfg dg dtdfg dg ddfg          d             Z_d Z`d Zad Zbd Zcd Zdd Zed Zf	 ddZgej:        ;                    deaeeedebf          ej:        ;                    de!eff          d                         Zhd Ziej:        ;                    dd          ej:        ;                    dd          ej:        ;                    d ejdM                    d                                     Zkd Zld Zmd Znej:        ;                    dg d}g dfg d}g dgfg d}gg dfg          d             Zod Zpd Zqd Zrd Zsd Ztd Zud Zvd Zwd Zxej:        ;                    dd'd g          d             Zyd Zzd Z{d Z|ej:        ;                    dg dQdMdfg dQddafg dQddfg          dÄ             Z}ej:        ;                    d ejH        g dŢ          dMdMf ejH        g dƢ          dMdaf ejH        g dƢ          ddMf ejH        g dǢ          dMdMf ejH        g dȢ          dMdaf ejH        g dȢ          ddMfg          dɄ             Z~ej:        ;                    d ejH        g dˢ          dg dQf ejH        g d̢          dag dQf ejH        g d͢          dag dQf ejH        g d΢          dg dϢfg          ej:        ;                    dd'd g          dф                         Zd҄ Zej:        ;                    dg dQdMdfg dQddafg dQddMfg          dӄ             Zej:        ;                    dg dQdfg dQdfg          dք             Zej:        ;                    dg dآg d٢g dڢg dۢg dܢgddfg dQg d٢g dڢg dۢg dܢgddfg dߢg d٢g dڢg dۢg dܢgg d?dfg dߢg d٢g dڢg dۢg dܢgg dKdfg dg d٢g dڢg dۢg dܢgg dQdfg dg d٢g dڢg dۢg dܢgg ddfd dMgg dg dgddfg          d             Zd ZdS )    N)
csr_matrix)stats)datasets)svm)softmax)make_multilabel_classification)_sparse_random_matrix)check_arraycheck_consistent_length)check_random_state)assert_allcloseassert_almost_equal)assert_array_equal)assert_array_almost_equal)accuracy_scoreauc)average_precision_score)coverage_error)	det_curve)%label_ranking_average_precision_score)precision_recall_curve)label_ranking_loss)roc_auc_score)	roc_curve)_ndcg_sample_scores_dcg_sample_scores)
ndcg_score	dcg_score)top_k_accuracy_score)UndefinedMetricWarning)train_test_split)LogisticRegression)label_binarizeFc                    | t          j                    } | j        }| j        }|r||dk              ||dk              }}|j        \  }}t          j        |          }t          d          }|                    |           ||         ||         }}t          |dz            }t
          j
                            d          }t
          j        ||                    |d|z            f         }t          j        ddd          }	|	                    |d|         |d|                                       ||d                   }
|r|
ddd	f         }
|	                    ||d                   }||d         }|||
fS )
zMake some classification predictions on a toy dataset using a SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    N   %   r      linearT)kernelprobabilityrandom_state   )r   	load_irisdatatargetshapenparanger   shuffleintrandomRandomStatec_randnr   SVCfitpredict_probapredict)datasetbinaryXy	n_samples
n_featuresprnghalfclfy_scorey_predy_trues                ^/var/www/html/Sam_Eipo/venv/lib/python3.11/site-packages/sklearn/metrics/tests/test_ranking.pymake_predictionrM   4   sy    $&&AA "Qx1q51GIz
	)A
R
 
 CKKNNNQ41qAy1}D )


"
"C
a9cJ&67778A 't!
D
D
DCggah%4%))77$%%AAG   !!!Q$-[[455""FtuuXF67""    c                 N   t          j        |           d         }|| |k             }|| |k             }|                    dd          |                    dd          z
  }t          j        |dk              }|t	          t          |          t          |          z            z  S )zKAlternative implementation to check for correctness of
    `roc_auc_score`.r.   r   )r3   uniquereshapesumfloatlen)rK   rI   	pos_labelposnegdiff_matrix	n_corrects          rL   _aucr[   d   s     	&!!!$I &I%
&C
&I%
&C++a$$s{{2q'9'99K{Q''IuSXXC01111rN   c                    t          j        |           d         }t          j        | |k              }t          j        |          ddd         }||         }| |         } d}t	          t          |                    D ]D}| |         |k    r6d}t	          d|dz             D ]}| |         |k    r|dz  }||dz   z  }||z  }E||z  S )a>  Alternative implementation to check for correctness of
    `average_precision_score`.

    Note that this implementation fails on some edge cases.
    For example, for constant predictions e.g. [0.5, 0.5, 0.5],
    y_true = [1, 0, 0] returns an average precision of 0.33...
    but y_true = [0, 0, 1] returns 1.0.
    r.   NrP   r         ?)r3   rQ   rS   argsortrangerU   )	rK   rI   rV   n_posorderscoreiprecjs	            rL   _average_precisionrf   s   s     	&!!!$IF6Y&''EJw"%EenGE]FE3w<<   	 	!9	!! D1a!e__    !9	))CKDAGODTME5=rN   c                 "   t          | |          \  }}}t          t          |                    }t          t          |                    }d}t          dt	          |                    D ]"}|||         ||         ||dz
           z
  z  z  }#|S )ao  A second alternative implementation of average precision that closely
    follows the Wikipedia article's definition (see References). This should
    give identical results as `average_precision_score` for all inputs.

    References
    ----------
    .. [1] `Wikipedia entry for the Average precision
       <https://en.wikipedia.org/wiki/Average_precision>`_
    r   r.   )r   listreversedr_   rU   )rK   rI   	precisionrecall	thresholdaverage_precisionrc   s          rL   _average_precision_slowrn      s     $:&'#J#J IvyXi(())I(6""##F1c)nn%% H HYq\VAYA-FGGrN   c                     d } || ||          \  }}t          ||          }d}|}d||z
  z  ||z   z  }	||z
  }
dd||	z
  |
|	z
  z  z   z  S )zcAlternative implementation to check for correctness of `roc_auc_score`
    with `max_fpr` set.
    c                 X   t          | |          \  }}}|||k             }t          j        ||          }|||k             }t          j        ||k              }|dz
  }	||	         ||         g}
||	         ||         g}t          j        |t          j        ||
|                    }||fS )Nr.   )r   r3   appendargmaxinterp)rK   	y_predictmax_fprfprtpr_new_fprnew_tpridx_outidx_inx_interpy_interps               rL   _partial_rocz,_partial_roc_auc_score.<locals>._partial_roc   s    	22S!cWn%)GW--cWn%)C'M**1KW.KW.)GRYw(%K%KLL!!rN   r         ?r.   r   )rK   rt   ru   r   ry   rz   partial_aucfpr1fpr2min_areamax_areas              rL   _partial_roc_auc_scorer      s    

" 
" 
" $|FIw??GWgw''K DDdTk"dTk2Hd{H!{X-(X2EFFGGrN   dropTc                 B   t          d          \  }}}t          ||          }t          |||           \  }}}t          ||          }t	          ||d           t          |t          ||                     |j        |j        k    sJ |j        |j        k    sJ d S )NTr@   drop_intermediater'   decimal)rM   r[   r   r   r   r   r   r2   )	r   rK   rx   rI   expected_aucrv   rw   
thresholdsroc_aucs	            rL   test_roc_curver      s     )555FAw((L$VWMMMCj#smmGg|Q????vw!?!?@@@9	!!!!9
(((((((rN   c                  \   t           j                            d          } t          j        dgdz  dgdz  z             }|                     dd          }t          ||d          \  }}}|d         dk    sJ |d	         dk    sJ |j        |j        k    sJ |j        |j        k    sJ d S )
Nr   2   r.      d   sizeTr   rP   )r3   r7   r8   arrayrandintr   r2   )rF   rK   rJ   rv   rw   thrs         rL   test_roc_curve_end_pointsr      s     )


"
"CXqcBh!r)**F[[[%%FffEEEMCcq6Q;;;;r7a<<<<9	!!!!9	!!!!!!rN   c                  d   t          d          \  } }}t          | |          \  }}}g }|D ]L}t          j        ||k    | z            }t          j        |           }	|                    d|z  |	z             Mt          ||d           |j        |j        k    sJ |j        |j        k    sJ d S )NTr   r]   r'   r   )rM   r   r3   rS   rq   r   r2   )
rK   rx   rI   rv   rw   r   tpr_correctttprE   s
             rL   test_roc_returns_consistencyr      s     )555FAw$VW55Cj K ) )VW\V+,,F6NN38a<(((( c;::::9	!!!!9
(((((((rN   c                      t          d          \  } }}t          j        t                    5  t	          | |           d d d            d S # 1 swxY w Y   d S )NFr   )rM   pytestraises
ValueErrorr   )rK   rx   rI   s      rL   test_roc_curve_multir      s    (666FAw	z	"	" # #&'"""# # # # # # # # # # # # # # # # # #s   AAAc                      t          d          \  } }}t          | |dz
            \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ d S )NTr   r   ?r'   r   )rM   r   r   r   r2   )rK   rx   rI   rv   rw   r   r   s          rL   test_roc_curve_confidencer      s    (555FAw$VWs];;Cj#smmGgtQ77779	!!!!9
(((((((rN   c                     t          d          \  } }}t          j        | j                  }t	          | |          \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ t          j        | j                  }t	          | |          \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ t	          | |          \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ d S )NTr   r   r'   r   g(\?)rM   r3   onesr2   r   r   r   zeros)rK   predrI   trivial_predrv   rw   r   r   s           rL   test_roc_curve_hardr     sp   +4888FD' 76<((L$V\::Cj#smmGgtQ77779	!!!!9
((((( 8FL))L$V\::Cj#smmGgtQ77779	!!!!9
((((( %VT22Cj#smmGgtQ77779	!!!!9
(((((((rN   c                     g d} g d}d}t          j        t          |          5  t          | |          \  }}}d d d            n# 1 swxY w Y   t	          |t          j        t          |          t
          j                             |j	        |j	        k    sJ |j	        |j	        k    sJ d}t          j        t          |          5  t          d | D             |          \  }}}d d d            n# 1 swxY w Y   t	          |t          j        t          |          t
          j                             |j	        |j	        k    sJ |j	        |j	        k    sJ d S )N)
r.   r.   r.   r.   r.   r.   r.   r.   r.   r.   )
r   r.   r   r.   r   r.   r   r.   r   r.   INo negative samples in y_true, false positive value should be meaninglessmatchHNo positive samples in y_true, true positive value should be meaninglessc                     g | ]}d |z
  S )r.    ).0xs     rL   
<listcomp>z,test_roc_curve_one_label.<locals>.<listcomp>1  s    )@)@)@A!a%)@)@)@rN   )
r   warnsr"   r   r   r3   fullrU   nanr2   )rK   rJ   expected_messagerv   rw   r   s         rL   test_roc_curve_one_labelr     s   +++F+++F 	T  
,4D	E	E	E 9 9(88S*9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 sBGC
OORV<<===9	!!!!9
((((( 	S  
,4D	E	E	E J J()@)@)@)@)@&IIS*J J J J J J J J J J J J J J J sBGC
OORV<<===9	!!!!9
(((((((s#   AAAC88C<?C<c                     ddg} ddg}t          | |          \  }}}t          | |          }t          |g d           t          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |g d           t          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |ddg           t          |ddg           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |g d           t          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |ddg           t          |ddg           t          |d           ddg} dd	g}d
}t	          j        t          |          5  t          | |          \  }}}d d d            n# 1 swxY w Y   t	          j        t                    5  t          | |           d d d            n# 1 swxY w Y   t          |g d           t          |t          j
        t          j
        t          j
        g           ddg} dd	g}d}t	          j        t          |          5  t          | |          \  }}}d d d            n# 1 swxY w Y   t	          j        t                    5  t          | |           d d d            n# 1 swxY w Y   t          |t          j
        t          j
        t          j
        g           t          |g d           t          j        ddgddgg          } t          j        ddgddgg          }t	          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t	          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t          t          | |d          d           t          t          | |d          d           t          j        ddgddgg          } t          j        ddgddgg          }t	          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t	          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t          t          | |d          d           t          t          | |d          d           t          j        ddgddgg          } t          j        ddgddgg          }t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          j        ddgddgg          } t          j        ddgddgg          }t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           d S )Nr   r.   r   r   r.   r   r.   r.   r]           r         ?      ?r   r   r   r   r]   r   macroaverageweightedsamplesmicro)r   r   r   r   r   r   r"   r   r   r3   r   r   )rK   rI   rw   rv   rx   r   r   s          rL   test_roc_curve_toydatar   8  s   VF!fGFG,,KCaFG,,Gc999---c999---%%%VF!fGFG,,KCaFG,,Gc999---c999---%%%VF!fGFG,,KCaFG,,GcAq6***cAq6***%%%VF!fGFG,,KCaFG,,Gc999---c999---%%%VFCjGFG,,KCaFG,,GcAq6***cAq6***%%%VFTlG 	S  
,4D	E	E	E 1 100S!1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 
z	"	" ' 'fg&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' 'c???333cBFBFBF#;<<<VFTlG 	T  
,4D	E	E	E 1 100S!1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 
z	"	" ' 'fg&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' 'cBFBFBF#;<<<c???333 X1v1v&''FhAA'((G	z	"	" 8 8fgw77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	z	"	" ; ;fgz::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;fgyIII3OOOfgwGGGMMMX1v1v&''FhAA'((G	z	"	" 8 8fgw77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	z	"	" ; ;fgz::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;fgyIII3OOOfgwGGGMMMX1v1v&''FhAA'((GfgwGGGKKKfgzJJJANNNfgyIII1MMMfgwGGGKKKX1v1v&''Fhc
S#J/00GfgwGGGMMMfgzJJJCPPPfgyIII3OOOfgwGGGMMMMMs   H''H+.H+I((I,/I,K<<L L  L==MMO77O;>O;P::P>P>S11S58S5T44T8;T8c                      g d} g d}t          | |d          \  }}}t          |g d           g d} g d}t          | |d          \  }}}t          |g d           d S )	Nr   r   r   r   r.   r.   )r   皙?r   333333?ffffff?r]   Tr   )       @r]   r   r   )r   r   r   r   r   r   r   r.   r.   r.   r.   r.   r.   )r   皙?r   r   r   皙?r   r   r   r   r   r   r]   )r   r]   r   r   r   r   )r   r   )rK   rI   rw   rv   r   s        rL    test_roc_curve_drop_intermediater     s    F,,,G$VWMMMCjj*>*>*>??? 544FOOOG$VWMMMCjj*H*H*HIIIIIrN   c                  ,   g d} g d}t          j        dd          }t          | ||          \  }}}t          j        |          dk                                     dk    sJ t          j        |          dk                                     dk    sJ d S )Nr   r   r.   r.   r.   )r   r   333333?皙?r   r      sample_weightr   )r3   repeatr   diffrS   )rK   rI   r   rv   rw   rx   s         rL   !test_roc_curve_fpr_tpr_increasingr     s     __F'''GIc1%%MFG=IIIKCaGCLL1!!##q((((GCLL1!!##q((((((rN   c                     ddg} ddg}t          t          | |          d           ddg} ddg}t          t          | |          d           g d} g d}t          t          | |          d           ddg} ddg}t          t          | |          d           g d} g d}t          t          | |          d           d S )Nr   r.   r   r.   r   r   r   r   r   r.   )r   r   )r   rB   s     rL   test_aucr     s    	
AA	
AAc!Qii---	
AA	
AAc!Qii---		A		Ac!Qii---	
AA	
AAc!Qii+++AAc!Qii-----rN   c                      t          j        t                    5  t          g dddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          dgdg           d d d            n# 1 swxY w Y   g d} g d}d                    t          j        |                     }t          j        t          t          j        |                    5  t          | |           d d d            d S # 1 swxY w Y   d S )	Nr   r   r   r   )r'   r.   r      )r            z+x is neither increasing nor decreasing : {}r   )	r   r   r   r   formatr3   r   reescape)r   rB   error_messages      rL   test_auc_errorsr     s   	z	"	" ) )OOOc3Z((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) 
z	"	"  SEC5               	AAAHHRSUUM	z=)A)A	B	B	B  Aq			                 s1   ;??A>>BB%DD
Dzy_true, labels)r   r.   r   r'   r   r.   r'   )abr   c)r   r   r   c           	         t          j        g dg dg dg dg          }t          g dg d          }t          g dg d          }||z   d	z  }t          g d
g d          }t          g dg d          }||z   d	z  }t          ddgddg          }	t          ddgddg          }
|	|
z   d	z  }||z   |z   dz  }t          t          | ||d          |           |||g}g d}t          j        ||          }t          t          | ||dd          |           d}t          j        t          |          5  t          | ||dd            d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   r   r   ffffff?r   333333?r   r   r   r.   r   r.   )r   r   r   r   r.   r   )r   r   r   r'   r.   r.   r   )r   r   r   r   )r   r   r   r.   r   r   r   r   r   r   ovolabelsmulti_class)r   r   r   )weightsr   r   r   r   z6average=None is not implemented for multi_class='ovo'.r   )r3   r   r   r   r   r   r   NotImplementedError)rK   r   y_scoresscore_01score_10average_score_01score_02score_20average_score_02score_12score_21average_score_12ovo_unweighted_scorepair_scores
prevalenceovo_weighted_scorer   s                    rL   #test_multiclass_ovo_roc_auc_toydatar    sQ    x	///+<+<+<mmmL H YYY(8(8(899HYYY88H 8+q0 YYY77HYYY(8(8(899H 8+q0 aVc3Z00HaVc3Z00H 8+q0 -/??BRRVWWfhv5III   $%57GHK###JKDDDHV
	
 	
 	
 		   MM	*-	@	@	@ X Xfhv5RVWWWWX X X X X X X X X X X X X X X X X Xs   ;EE!$E!)r   r'   r   r'   )r   dr   r  )r   r   r  c           	      ,   t          j        g dg dg dg dg          }t          g dg d          }t          g dg d          }||z   d	z  }t          t          | ||d
          |           t          t          | ||d
d          |           d S )N)r   r   r   )r   r   r   )皙?r   ?)r   r   r   )r.   r   r.   r   )r   r   r  r   )r   r.   r   r.   )r   r   r  r   r'   r   r   r   r   r3   r   r   r   )rK   r   r  r  r  	ovo_scores         rL   *test_multiclass_ovo_roc_auc_toydata_binaryr    s     x	///+<+<+<oooN H \\\+@+@+@AAH\\\+@+@+@AAHH$)Ifhv5III9  
 HV
	
 	
 	
 		    rN   )r   r.   r'   r'   )r   r   r   r   c           	         t          j        g dg dg dg dg          }t          g d|d d df                   }t          g d|d d df                   }t          g d	|d d d
f                   }t          t          | |d|d           |||g           ||z   |z   dz  }t          t          | |d|          |           |dz  |dz  z   |dz  z   }t          t          | |d|d          |           d S )Nr]   r   r   )r   r   r   )r   r   r   )r   r   r   r.   r   r   r   r   )r   r.   r   r   r.   r   r   r.   r.   r'   ovr)r   r   r   g      @)r   r   r   r   r   r  )rK   r   r  out_0out_1out_2result_unweightedresult_weighteds           rL   #test_multiclass_ovr_roc_auc_toydatar"  =  s]    x	///???OOOL H
 ,,,A77E,,,A77E,,,A77EfhE&RVWWW	u   .#5fhE&III   dlUT\1ECK?OH%
	
 	
 	
 		    rN   zmulti_class, average))r  r   )r  r   )r   r   c                 b   t          j        g d          }g dg dg dg dg}t          t          ||| |          d           g dg dg dg dg}t          ||| |          dk     sJ dt          j        d	          z  }t          ||| |          t          j        d
          k    sJ d S )N)r   r.   r'   r   )r   r   r   r]   )r   r]   r   r   )r   r   r]   r   )r   皙?r$  r   r   r   r]   r   )r   r   r   )r3   r   r   r   r   r   approx)r   r   rK   	y_perfecty_imperfecty_chances         rL   0test_perfect_imperfect_chance_multiclass_roc_aucr*  h  s#    Xlll##F 	   	I fi['RRR   		K 	fk{GTTT
	 	 	 	
 bgfoo%Hk7  	s		     rN   c                   	 | 	t           j                            g dd	          }t          j        	fd|D                       }t          |g d          }t          |                                |                                          \  }}}t          ||          }t          ||dd	          }|t          j        |          k    sJ d S )
N)r   r]   r     )r   r-   c                 x    g | ]6}t           j                            d |                                          7S )r.   )nrE   r-   )r   multinomialrvsrr   )r   y_pred_iseeds     rL   r   z3test_micro_averaged_ovr_roc_auc.<locals>.<listcomp>  sM     	
 	
 	
 !!A!EELLNN	
 	
 	
rN   r   )classesr  r   r%  )r   	dirichletr0  r3   asarrayr%   r   ravelr   r   r   r&  )
global_random_seedrJ   rK   y_onehotrv   rw   rx   roc_auc_by_handroc_auc_autor2  s
            @rL   test_micro_averaged_ovr_roc_aucr;    s    D
 _  t$ OOFZ	
 	
 	
 	
"	
 	
 	
 F fiii888HHNN,,fllnn==KCa#smmO UGTTTLfmL99999999rN   zmsg, y_true, labelsz!Parameter 'labels' must be uniquer   r'   r   )r   r   r   zKNumber of classes in y_true not equal to the number of columns in 'y_score'z"Parameter 'labels' must be ordered)r   r   r   zMNumber of given labels, 2, not equal to the number of columns in 'y_score', 3r.   r   r   zMNumber of given labels, 4, not equal to the number of columns in 'y_score', 3)r   r.   r'   r   )r   r   r   r  z2'y_true' contains labels not in parameter 'labels')r   r   r   er   r   r  c                     t          j        g dg dg dg dg          }t          j        t          |           5  t          ||||           d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   r   r   )r3   r   r   r   r   r   )msgrK   r   r   r  s        rL   *test_roc_auc_score_multiclass_labels_errorr@    s    ~ x	///+<+<+<mmmL H 
z	-	-	- P Pfhv;OOOOP P P P P P P P P P P P P P P P P Ps   AA!$A!zmsg, kwargszLaverage must be one of \('macro', 'weighted', None\) for multiclass problemsr   )r   r   zUaverage must be one of \('micro', 'macro', 'weighted', None\) for multiclass problemszksample_weight is not supported for multiclass one-vs-one ROC AUC, 'sample_weight' must be None in this case)r   r   z|Partial AUC computation not available in multiclass setting, 'max_fpr' must be set to `None`, received `max_fpr=0.5` insteadr   )r   ru   zbmulti_class='ovp' is not supported for multiclass ROC AUC, multi_class must be in \('ovo', 'ovr'\)ovpz'multi_class must be in \('ovo', 'ovr'\)c                 "   t          d          }|                    dd          }t          |          }|                    ddd          }t	          j        t          |           5  t          ||fi | d d d            d S # 1 swxY w Y   d S )N     r   r   r   r   )r   randr   r   r   r   r   r   )r?  kwargsrF   rI   y_probrK   s         rL   #test_roc_auc_score_multiclass_errorrH    s    \ S
!
!Chhr1ooGWF[[AB[''F	z	-	-	- 0 0ff/////0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   (BBBc                  &   t          d          } |                     d          }t          j        dd          }d}t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j        dd          }t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j	        ddd          }t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j        d	          5  t          d          } |                     d          }t          j        dd          }t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j        dd          }t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j	        ddd          }t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )
NrC  
   r6   dtypezROC AUC score is not definedr   rP   T)record)r   rE  r3   r   r   r   r   r   r   r   warningscatch_warnings)rF   rJ   rK   err_msgs       rL   test_auc_score_non_binary_classrQ  #  s    S
!
!CXXb\\FXb&&&F,G	z	1	1	1 & &ff%%%& & & & & & & & & & & & & & &WRu%%%F	z	1	1	1 & &ff%%%& & & & & & & & & & & & & & &WR5)))F	z	1	1	1 & &ff%%%& & & & & & & & & & & & & & & 
	 	-	-	- * * %%""E***]:W555 	* 	*&&)))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	*5)))]:W555 	* 	*&&)))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	*Ru---]:W555 	* 	*&&)))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	** * * * * * * * * * * * * * * * * *s   A55A9<A91CCCD((D,/D,AJF;/J;F?	?JF?	4J7HJH	JH	5JI."J.I2	2J5I2	6JJ
J

curve_funcc                    t          d          }|                    ddd          }|                    d          }d}t          j        t
          |          5   | ||           d d d            d S # 1 swxY w Y   d S )NrC  r   r   rJ  r   z"multiclass format is not supportedr   )r   r   rE  r   r   r   )rR  rF   rK   rJ   r?  s        rL   &test_binary_clf_curve_multiclass_errorrT  C  s    
S
!
!C[[AB[''FXXb\\F
.C	z	-	-	- # #
66"""# # # # # # # # # # # # # # # # # #s   A44A8;A8c                    d}t          j        t          |          5   | t          j        ddgd          ddg           d d d            n# 1 swxY w Y   t          j        t          |          5   | t          j        ddgt
                    ddg           d d d            n# 1 swxY w Y   d	}t          j        t          |          5   | t          j        d
dgd          ddg           d d d            n# 1 swxY w Y   g d} | g d|          } | g d|          }t          ||          D ]%\  }}t          j                            ||           &d S )Nzy_true takes value in {'a', 'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.r   r   r   z<U1rK  r   r]   zy_true takes value in {b'a', b'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.   a   bz<S1)r   r]   r   gzG?)r   r.   r.   r   )r   r]   r]   r   )	r   r   r   r3   r   objectziptestingr   )rR  r?  rJ   	int_curvefloat_curveint_curve_partfloat_curve_parts          rL   (test_binary_clf_curve_implicit_pos_labelr_  M  s   
	  
z	-	-	- B B
28S#Je444sCjAAAB B B B B B B B B B B B B B B 
z	-	-	- C C
28S#Jf555SzBBBC C C C C C C C C C C C C C C	  
z	-	-	- D D
28T4L666c
CCCD D D D D D D D D D D D D D D
 #""F
<<<00I*1116::K,/	;,G,G E E((

"">3CDDDDE Es5   %AAA5*B++B/2B/%DDDc                     g d}g d}g d} | |||          } | |d d         |d d         |d d                   }t          ||          D ]\  }}t          ||           d S )Nr   )r   r   r   r   r   )r.   r.   r.   r   r   r   rP   )rY  r   )rR  rK   rI   r   result_1result_2arr_1arr_2s           rL   (test_binary_clf_curve_zero_sample_weightre  q  s    __F'''G%%%Mz&'GGGHz&"+wss|=QTRTQTCUVVVHHh// & &uu%%%%& &rN   c                     t          d          \  } }}t          | |           t          | dd          |dd                    \  }}}|d         dk    sJ |d         | dd                                          k    sJ d| t	          j        | dk              <   |                                 }t          | |           t          ||            g d}g d}t          ||          \  }}}t          |t	          j	        g d	                     t          |t	          j	        g d
                     t          |t	          j	        g d                     |j
        |j
        k    sJ |j
        |j
        dz   k    sJ d S )NTr   r.   r   r]   rP   r.   r   r   r.   )r.   r'   r   r   )r   gQUU?r   r]   r]   )r]   r   r   r   r   )rM   _test_precision_recall_curver   meanr3   wherecopyr   r   r   r   )	rK   rx   rI   rE   rr   y_true_copyr   predict_probass	            rL   test_precision_recall_curvero  ~  s   (555FAw 111 %VABBZ==GAq!Q43;;;;Q46!"":??$$$$$$ %'F28FaK  !++--K 111{F+++\\F!\\N$V^<<GAq!a*J*J*J!K!KLLLa*C*C*C!D!DEEEa,,,!7!78886QV6QVaZrN   c                    t          | |          \  }}}t          | |          }t          |dd           t          |t          | |                     t	          t          | |          |d           |j        |j        k    sJ |j        |j        dz   k    sJ t          | t          j        |                    \  }}}|j        |j        k    sJ |j        |j        dz   k    sJ d S )Ngrh|?r   r'   r   r.   )	r   rn   r   r   r   rf   r   r3   
zeros_like)rK   rI   rE   rl  r   precision_recall_aucs         rL   rh  rh    s   -fg>>Aq*267CC2E1===5fgFF   67++-A1    6QV6Z_q(((((-fbmG6L6LMMAq*6QV6Z_q(((((((rN   c            	         t          j        d          5  ddg} ddg}t          | |          \  }}}t          | |          }t	          |g d           t	          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t	          |g d           t	          |g d	           t          |d
           ddg} ddg}t          | |          \  }}}t          | |          }t	          |d
dg           t	          |ddg           t          |d
           ddg} ddg}t          | |          \  }}}t          | |          }t	          |g d           t	          |g d           t          |d           ddg} d
d
g}t          | |          \  }}}t          | |          }t	          |d
dg           t	          |ddg           t          |d
           ddg} ddg}t          j        t          d          5  t          | |          \  }}}d d d            n# 1 swxY w Y   t          j        t          d          5  t          | |          }d d d            n# 1 swxY w Y   t          |g d           t          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          t          | |          d           t	          |g d           t	          |g d           t          j
        ddgddgg          } t          j
        ddgddgg          }t          j        t          d          5  t          t          | |d          d
           d d d            n# 1 swxY w Y   t          j        t          d          5  t          t          | |d          d           d d d            n# 1 swxY w Y   t          t          | |d          d           t          t          | |d          d           t          j
        ddgddgg          } t          j
        ddgddgg          }t          j        t          d          5  t          t          | |d          d
           d d d            n# 1 swxY w Y   t          j        t          d          5  t          t          | |d          d           d d d            n# 1 swxY w Y   t          t          | |d          d           t          t          | |d          d
           t          j
        ddgddgg          } t          j
        ddgddgg          }t          t          | |d          d
           t          t          | |d          d
           t          t          | |d          d
           t          t          | |d          d
           t          j
        ddgddgg          } t          j
        ddgddgg          }t          j        t          d          5  t          t          | |d          d           d d d            n# 1 swxY w Y   t          t          | |d          d           t          j        t          d          5  t          t          | |d          d           d d d            n# 1 swxY w Y   t          j        t          d          5  t          t          | |d          d           d d d            n# 1 swxY w Y   t          j
        ddgddgg          } t          j
        ddgddgg          }t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          j
        ddgddgg          } t          j
        d
d
gd
d
gg          }t          t          | |d          d
           t          t          | |d          d
           t          t          | |d          d
           t          t          | |d          d
           d d d            n# 1 swxY w Y   t          j        d          5  t          j
        ddgddgg          } t          j
        ddgddgg          }t          j        t          d          5  t          t          | |d          d           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nraise)allr   r.   )r   r.   r.   r   r]   )r   r   r]   r  r   r   r   r   z!No positive class found in y_truer   r   )r]   r]   r]   )r.   r   r   r   r   r   r   r   ignore)r3   errstater   r   r   r   r   r   UserWarningr   r   )rK   rI   rE   rl  rx   auc_prcs         rL   #test_precision_recall_curve_toydatarz    sv   		!	!	! V
 V
Qa&(991a)&'::!![[[111!!YYY///GS)))Qa&(991a)&'::!!___555!!___555 	GS)))Qa&(991a)&'::!!c1X...!!c1X...GS)))Qa&(991a)&'::!![[[111!!YYY///GS)))Q*(991a)&'::!!c1X...!!aX...GS)))Q,\+-PQQQ 	> 	>,VW==GAq!	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	>\+-PQQQ 	? 	?-fg>>G	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	?999%%%999%%%###Q,(991a3FGDDcJJJ!!___555!!]]]333 Aq6Aq6*++(QFQF+,,\+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 \+-PQQQ 	 	'LLLc  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	#FGYGGG	
 	
 	
 	/QQQSVWWWAq6Aq6*++(QFQF+,,\+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 \+-PQQQ 	 	'LLLc  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	#FGYGGG	
 	
 	
 	/QQQSVWWWAq6Aq6*++(QFQF+,,#FGWEEEs	
 	
 	
 	#FGZHHH#	
 	
 	
 	#FGYGGG	
 	
 	
 	#FGWEEEs	
 	
 	
 Aq6Aq6*++(QFQF+,,\+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	#FGZHHH#	
 	
 	
 \+-PQQQ 	 	'KKKS  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 \+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	
 Aq6Aq6*++(QFQF+,,/QQQSVWWW#FGZHHH#	
 	
 	
 	#FGYGGG	
 	
 	
 	/QQQSVWWWAq6Aq6*++(S#Jc
344#FGWEEEs	
 	
 	
 	#FGZHHH#	
 	
 	
 	#FGYGGG	
 	
 	
 	#FGWEEEs	
 	
 	
iV
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
p 
	"	"	"  Aq6Aq6*++(QFQF+,,\+-PQQQ 	 	'LLLa  	 	 	 	 	 	 	 	 	 	 	 	 	 	 		                 s  HcH;/c;H?	?cH?	c!I>2c>J	cJ	C$c*!NcN	cN	c=!O*c*O.	.c1O.	2Bc!R1%c1R5	5c8R5	9c!T8cT	cT	Ec!Y?3c?Z	cZ	>c![2&c2[6	6c9[6	:c!]9c]			c]		E+ccc$Ae94!e!e9!e%	%e9(e%	)e99e= e=c                      t          j        dt                    } d| d d d<   t          j        d          }t	          | |          dk    sJ d S )Nr   rK  r.   r   r   )r3   r   r6   r   r   rK   rI   s     rL   &test_average_precision_constant_valuesr}  O  sW    
 Xc%%%FF33Q3KgcllG #6733t;;;;;;rN   c                     t          j        ddg          } t          j        ddg          }d}t          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   t          j        ddgddgddgddgg          } t          j        ddgddgd	d
gd
d	gg          }d}t          j        t          |          5  t          | |d           d d d            d S # 1 swxY w Y   d S )Nr   r.   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r   r'   rV   r   r   r   r   znParameter pos_label is fixed to 1 for multilabel-indicator y_true. Do not set pos_label or set pos_label to 1.)r3   r   r   r   r   r   rK   rJ   rP  s      rL   -test_average_precision_score_pos_label_errorsr  ]  s   Xq!fFXq!fFOG	z	1	1	1 = =!<<<<= = = = = = = = = = = = = = = X1v1v1v1v677FXSzC:SzC:FGGF	6  
z	1	1	1 = =!<<<<= = = = = = = = = = = = = = = = = =s$   
A))A-0A-C11C58C5c                     t          d          \  } }}t          | |          }t          | d|z            }t          | d|z            }t          | |dz
            }||k    sJ ||k    sJ ||k    sJ t          | |          }t          | d|z            }t          | d|z            }	t          | |dz
            }
||k    sJ ||	k    sJ ||
k    sJ d S )NTr   r   gư>rJ  )rM   r   r   )rK   rx   rI   r   roc_auc_scaled_uproc_auc_scaled_downroc_auc_shiftedpr_aucpr_auc_scaled_uppr_auc_scaled_downpr_auc_shifteds              rL   test_score_scale_invariancer  p  s    )555FAwFG,,G%fcGm<<'w??#FGbL99O''''')))))o%%%%$VW55F.vsW}EE0HH,VWr\BBN%%%%%'''''^######rN   z(y_true,y_score,expected_fpr,expected_fnrr   r   )r   r   r   )r   r   r.   r   r   r   r   r   r   r   )r.   r.   r   r   r   )r   r   r   c                 n    t          | |          \  }}}t          ||           t          ||           d S Nr   r   rK   rI   expected_fprexpected_fnrrv   fnrrx   s          rL   test_det_curve_toydatar    s?    6 FG,,KCaC&&&C&&&&&rN   r   r   r   c                 n    t          | |          \  }}}t          ||           t          ||           d S r  r  r  s          rL   test_det_curve_tie_handlingr    s?     FG,,KCaC&&&C&&&&&rN   c                  n    t          t          g dg d          t          g dg d                     d S )Nr   r   r   )r   r   r   r   r.   r.   )r   r   r   rN   rL   test_det_curve_sanity_checkr    sN    )))[[[))$$$&<&<&<==    rN   rI   )r   r   r   r   r.   c                     t          g dt          j        d|                     \  }}}t          |dg           t          |dg           t          || g           d S )N)r   r.   r   r.   r   r.   r   r|  r.   r   )r   r3   r   r   )rI   rv   r  rl   s       rL   test_det_curve_constant_scoresr    sr    #!!!271g+>+>  Ci C!C!Iy)))))rN   rK   )r   r   r   r   r   r.   r   )r   r   r   r.   r.   r.   )r   r   r.   r.   r.   r.   )r   r.   r.   r.   r.   r.   c                 t    t          | |           \  }}}t          |dg           t          |dg           d S )Nr|  r   r  )rK   rv   r  rx   s       rL   test_det_curve_perfect_scoresr    sF     66:::KCaC!C!rN   zy_true, y_pred, err_msgzinconsistent numbers of samplesr   r   r   z Only one class present in y_truer.   r.   r.   )cancerr  
not cancer)r   r   r   zpos_label is not specifiedc                     t          j        t          |          5  t          | |           d d d            d S # 1 swxY w Y   d S )Nr   )r   r   r   r   r  s      rL   test_det_curve_bad_inputr    s      
z	1	1	1 " "&&!!!" " " " " " " " " " " " " " " " " "   :>>c                     dgdz  dgdz  z   } t          j        g d          }d|z
  }t          | |d          \  }}}t          | |d          \  }}}|d         t          j        d	          k    sJ |d         t          j        d
          k    sJ t          ||d d d                    t          ||d d d                    d S )Nr  r   r  r   )
r   r   r   r   r   r   r   r   r   r   r.   r  r   r   r   rP   )r3   r   r   r   r&  r   )	rK   y_pred_pos_not_cancery_pred_pos_cancerfpr_pos_cancerfnr_pos_cancerth_pos_cancerfpr_pos_not_cancerfnr_pos_not_cancerth_pos_not_cancers	            rL   test_det_curve_pos_labelr    s   Z!^|nq00FH%W%W%WXX114=5 5 51NNM
 AJA A A=*,= v}S111111Q6=#5#55555 N$6ttt$<===N$6ttt$<=====rN   c                    t           | ddggddgg          d           t           | ddggddgg          d           t           | ddggddgg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | ddggddgg          d           t           | ddggddgg          d           t           | ddggddgg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           d S )Nr   r.   r   r   r   r   r  r   r   r   gUUUUUU?r   g?r   g?r   r   r   r  r   r   r   r  )r   r   r   UUUUUU?)r.   r.   r.   r   )r   r   r   r   r   
lrap_scores    rL   check_lrap_toyr    s/   

QF8tTl^<<a@@@

QF8tTl^<<eDDD

QF8tTl^<<a@@@

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDeLLL

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDeLLL
III;!2!2!2 3446I   
III;!2!2!2 3446I   

III;1B1B1B0CDDeLLL

III;1B1B1B0CDDeLLL
III;!2!2!2 3446I   

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDoVVV

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDeLLL

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDoVVV

III;1B1B1B0CDDeLLL
III;!2!2!2 3446I   

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDaHHH 

QF8sCj\::C@@@

QF8sCj\::C@@@

QF8sCj\::A>>>

III;1A1A1A0BCCSIII

III;1A1A1A0BCCSIII

III;1A1A1A0BCCQGGG

III;1A1A1A0BCCUKKK
III;!1!1!1 2335H   
III;!1!1!1 2335H   

III;1A1A1A0BCCQGGG

III;0ABBEJJJ

LLL>4H4H4H3IJJERRRRRrN   c           	         t          d          }t          dd          D ]}|                    d|f          }t          j        |          }t          j        d|f          } | ||          dk    sJ  | ||          dk    sJ t          j        d|f          } | ||          dk    sJ  | ||          dk    sJ t           | dgdgdgdggdgdgdgdgg          d           d S )Nr   r'   r   r.   r   r]   r   )r   r_   uniformr3   rq  r   r   r   )r  r-   n_labelsrI   y_score_tiesrK   s         rL   !check_zero_or_all_relevant_labelsr  K  sA   %a((L!QKK 7 7&&QM&::}W-- 1h-((z&'**c1111z&,//36666 !X''z&'**c1111z&,//366666 
QC!qcA3'3%#u)EFF    rN   c                    t          j        t                    5   | g dg d           d d d            n# 1 swxY w Y   t          j        t                    5   | g dg dg dg dg           d d d            n# 1 swxY w Y   t          j        t                    5   | g dg dg dg dg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggddg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggddgg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggdgdgg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5   | dgdggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggdgdgg           d d d            d S # 1 swxY w Y   d S )	Nr   )r   r   r   r   )r   r   r   )r   r   r   )r   r   r   r   r.   )r   r   r   r  s    rL   check_lrap_error_raisedr  b  s   	z	"	" 0 0
999...///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0	z	"	" U U
999000///???STTTU U U U U U U U U U U U U U U	z	"	" 
 

OO//////R	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
z	"	" - -
QFQF#aV,,,- - - - - - - - - - - - - - -	z	"	" / /
QFQF#q!fX.../ / / / / / / / / / / / / / /	z	"	" 1 1
QFQF#qcA3Z0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1	z	"	" / /
QF8q!fq!f-.../ / / / / / / / / / / / / / /	z	"	" 1 1
QC!:AA/0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1	z	"	" 1 1
QFQF#qcA3Z0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s   7;;A??BB#CCC+DDD0EEE6FF F=GG#&G#H&&H*-H*
I..I25I2c           	      &   t          dd          D ]}t          j        d|f          }t          d|          D ]V}t          ||z
            D ]A}t          j        d|f          }d|d|||z   f<   t	           | ||          ||z             BWd S )Nr'   rJ  r.   r   )r_   r3   r   r   r   )r  r  rI   
n_relevantrW   rK   s         rL   check_lrap_only_tiesr  |  s     !RLL 	X 	X'1h-((  8,, 	X 	XJX
233 X X1h-0045q#j 0001#JJvw$?$?hAVWWWWX	X		X 	XrN   c                 .   t          dd          D ] }|t          j        |                              d|f          dz   z
  }t          j        d|f          }d|d<   d|d<   t           | ||          d|z  dz   dz             t          d|          D ]zt          |z
            D ]et          j        d|f          }d|dz   f<   t           | ||          t          fdt                    D                                  f{d S )Nr'   rJ  r.   )r   r   )r   rP   r   c              3   :   K   | ]}|d z   |z   d z   z  z  V  dS )r.   Nr   )r   rl  r  rW   s     rL   	<genexpr>z>check_lrap_without_tie_and_increasing_score.<locals>.<genexpr>  sM         QC!GaK:#=>     rN   )r_   r3   r4   rR   r   r   rS   )r  r  rI   rK   r  rW   s       @@rL   +check_lrap_without_tie_and_increasing_scorer    sv    !RLL  bi11991h-HH1LM 1h-((tuJJvw77!h,:Ja9OPPP  8,, 	 	JX
233 	 	1h-0045q#j 0001#Jvw//     !&z!2!2       		 rN   c                    t          | |           t          |           } t          |          }| j        \  }}t          j        |f          }t          |          D ]}t          j        ||         d          \  }}|j        }||z
  t          j        |dz             	                                }	|	         | |         
                                d         }
|
j        dk    s|
j        |k    rd||<   d||<   |
D ]7t          fd|
D                       }||xx         |         z  z  cc<   8||xx         |
j        z  cc<   |                                S )z8Simple implementation of label ranking average precisionT)return_inverser.   )	minlengthr   r   c              3   <   K   | ]}|                  k    V  d S r  r   )r   rl  labelranks     rL   r  z_my_lrap.<locals>.<genexpr>  s0       J JAaDK!7 J J J J J JrN   )r   r
   r2   r3   emptyr_   rQ   r   bincountcumsumnonzerorS   ri  )rK   rI   rC   r  rb   rc   unique_rankinv_rankn_ranks	corr_rankrelevantn_ranked_abover  r  s               @@rL   _my_lrapr    s   FG,,,  F'""G ,IxHi\""E9 " " !#	'!*T J J JX"! K!<<<CCEE	!9$$&&q)=A(!:!:E!Ha 	5 	5E ! J J J J J J J JJJN !HHHe44HHHHaHM!::<<rN   r   rD  c                    t          dd|||          \  }}t          |j        d         |j        d         |          }t          |d          r|                                }t          ||          }t          ||          }t          ||           t          |          }|	                    ||f          }t          ||          }t          ||          }t          ||           d S )Nr.   F)rD   allow_unlabeledr-   	n_classesrC   r   )n_componentsrD   r-   toarrayr   )
r   r	   r2   hasattrr  r   r  r   r   r  )	r  r  rC   r-   rx   rK   rI   
score_lrapscore_my_lraps	            rL   %check_alternative_lrap_implementationr    s    /!  IAv $\!_<?!  G w	"" $//##6vwGGJVW--M
M222 &l33L""I(>"??G6vwGGJVW--M
M22222rN   checkfuncc                      | |           d S r  r   )r  r  s     rL   test_label_ranking_avpr    s     
E$KKKKKrN   c                  .    t          t                     d S r  )r  r   r   rN   rL   test_lrap_error_raisedr    s    ABBBBBrN   rC   )r.   r'   r   rD  r  )r'   r   rJ  r-   c                 4    t          t          || |           d S r  )r  r   )rC   r  r-   s      rL   $test_alternative_lrap_implementationr    s)    
 *-y)\    rN   c                  t   t          j        g dg dg dgt                    } t          j        g dg dg dg          }t          j        g d          }t          j        g d	          }t          t	          | ||
          t          j        ||z            t          j        |          z             d S )Nr  rg  )r   r   r   r   rK  )r   r   r   r   )r   r   r   r   r   r   r   r   )r   r   r]   )r]   r]   r   r   )r3   r   boolr   r   rS   )rK   rI   samplewise_lrapsr   s       rL   &test_lrap_sample_weighting_zero_labelsr    s     X|||\\\<<<@MMMFh			3335I5I5IJ G x 0 0 011H___--M-G=	
 	
 	
 	}//0026-3H3HH	    rN   c            	         t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	g dgg dg dg          d           t          t          g d	g dg d
gg dg dg dg          d           t          t          g d	g dg d
gg dg dg dg          d           d S )Nr   r.   r   r   r'   r  r  r   r   r   r   r   r   r   r  r  r  r   g      $@r   r.   r   r   r   rJ  r  r<  g@r   r.   r   r   r   r   rN   rL   test_coverage_errorr    st   Ax4,@@!DDDAx4,@@!DDDAx4,@@!DDDAx4,@@!DDD			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL 			999-/KLL  
 YY			999-yyy)))/T	
 	
 		   YY			999-yyy)))/T	
 	
 		    rN   c                  f   t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           d S )Nr   r   r.   r'   r  r  r   r   r   r   r   r   r   r  r  r   rN   rL   test_coverage_tie_handlingr  T  s   Ax3*>>BBBAx3*>>BBBAx3*>>BBBAx3*>>BBB			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKKKKrN   zy_true, y_scorec                     t          j        t          d          5  t          | |           d d d            d S # 1 swxY w Y   d S )Nz'Expected 2D array, got 1D array insteadr   )r   r   r   r   r|  s     rL   test_coverage_1d_error_messager  d  s     
z)S	T	T	T ( (vw'''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (r  c                  ,   t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dg dgg dg dg          d           t          t          g dg dg d	gg dg dg dg          d           t          t          g dg dg d	gg dg dg dg          d           t          t          t          t          j        g dg dg                    g dg dg          d           d S )Nr   r.   r   r   r   r  r   r   r   r   r]   r   r   r  r  r  r  r  r  r  r<  r  )r   r   r   r3   r   r   rN   rL   test_label_ranking_lossr  s  s   *QF8tTl^DDaHHH*QF8tTl^DDaHHH*III;9J9J9J8KLLaPPP*III;9J9J9J8KLLeTTT*III;9J9J9J8KLLaPPP*III;9J9J9J8KLLeTTT*III;9J9J9J8KLLeTTT*III;9J9J9J8KLLeTTT *QF8tTl^DDaHHH*QF8tTl^DDaHHH*QF8sCj\BBAFFF*QF8sCj\BBAFFF*III;9J9J9J8KLLaPPP*III;9J9J9J8KLLaPPP*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOO IIIyyy1OOOYYY3OPP  
 YY			999-yyy)))/T	
 	
 	"	   YY			999-yyy)))/T	
 	
 	"	   rxIII 67788===))):T	
 	
 		    rN   c                     t          j        t                    5  t          ddgddggddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddgddggddgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddgddggdgdgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          dgdggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddgddggdgdgg           d d d            d S # 1 swxY w Y   d S )Nr   r.   )r   r   r   r   r   rN   rL   $test_ranking_appropriate_input_shaper    sE   	z	"	" 5 5QFQF+aV4445 5 5 5 5 5 5 5 5 5 5 5 5 5 5	z	"	" 7 7QFQF+q!fX6667 7 7 7 7 7 7 7 7 7 7 7 7 7 7	z	"	" 9 9QFQF+qcA3Z8889 9 9 9 9 9 9 9 9 9 9 9 9 9 9	z	"	" 7 7QF8q!fq!f%56667 7 7 7 7 7 7 7 7 7 7 7 7 7 7	z	"	" 9 9QC!:AA'78889 9 9 9 9 9 9 9 9 9 9 9 9 9 9	z	"	" 9 9QFQF+qcA3Z8889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9si   ?AA#B		BB-CCC8DD"%D"E))E-0E-F55F9<F9c                  F   t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           d S )Nr.   r   r   r   r  r   r   r   r   r   )r   r   r   rN   rL   test_ranking_loss_ties_handlingr    sX   *QF8sCj\BBAFFF*QF8sCj\BBAFFF*III;9I9I9I8JKKUSSS*III;9I9I9I8JKKUSSS*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOOOOrN   c                      t          dd          \  } }| dz   }t          ||           t          j                            d                              d          \  }}t          ||           d S Nr   rJ  )r-   r  r.   )r'   r   rJ  )r   _test_dcg_score_forr3   r7   r8   random_samplerx   rK   rI   s      rL   test_dcg_scorer    sp    .ALLLIAvgkG(((i++A..<<\JJOFG(((((rN   c                 :   t          j        t          j        | j        d                   dz             }t	          | |           }t	          | |          }||k                                    sJ t	          | | d          |k                                    sJ |j        | j        d         fk    sJ |j        | j        d         fk    sJ |t          j        t          j        |           d d d d df         |z  	                    d                    k    sJ d S )Nr.   r'   r   kr   rP   axis)
r3   log2r4   r2   r   ru  r   r&  sortrS   )rK   rI   discountidealrb   s        rL   r  r    s   wrya11A566Hvv..Evw//EUN!!!!!vv333u<AACCCCC;6<?,,,,,;6<?,,,,,FM276??111ddd7#;h#F"K"KQR"K"S"STTTTTTTTrN   c            	         t          j        t          j        d          g          } t          j        | j                  }t          | |          }t          | |d          }dt          j        t          j        dd                    z  }|t          j        |	                                | 
                                z  g          k    sJ |t          j        || d d d d df         z  	                                g          k    sJ d|dd	d f<   t          | |          }t          | |d          }|t          j        || d d d d df         z  	                                g          k    sJ |t          j        |d d         	                                | dd	d f         
                                z  |dd          	                                | dd d	f         
                                z  z   g          k    sJ d S )
Nr   Tignore_tiesr.   r'   r   rP   r   r   )r3   r5  r4   r   r2   r   r  r   r&  rS   ri  )rK   rI   dcgdcg_ignore_ties	discountss        rL   test_dcg_tiesr    s   Z1''Fhv|$$G
VW
-
-C(dKKKOBGBIaOO,,,I&-6;;==!@ ABBBBBBfmi&DDbD/.I-N-N-P-P,QRRRRRRGAqrrEN
VW
-
-C(dKKKOfmi&DDbD/.I-N-N-P-P,QRRRRRR&-bqbM&ABB-"4"4"6"66m!!F1bqb5M$6$6$8$889	
       rN   c            	          t          j        d                              d          } t          | | dd          t	          j        t          | | dd                    k    sJ d S )N   )r'   r   r   T)r  r  )r3   r4   rR   r   r   r&  )r   s    rL   test_ndcg_ignore_ties_with_kr    sq    
	"f%%AaaT222fm1a1$///7 7      rN   c                      t          j        g dg          } t          j        g dg          }d}t          j        t          |          5  t          | |          t          j        d          k    sJ 	 d d d            d S # 1 swxY w Y   d S )N)g{Gzg(\gGz޿g(\?gQ?)gQ?gףp=
?r   gQ?gHzG?zndcg_score should not be used on negative y_true values. ndcg_score will raise a ValueError on negative y_true values starting from version 1.4.r   g&x@)r3   r   r   r   FutureWarningr   r&  )rK   rI   r   s      rL   test_ndcg_negative_ndarray_warnr    s    X8889::Fh666788G	M  
m+;	<	<	< F F&'**fmH.E.EEEEEEF F F F F F F F F F F F F F F F F Fs   )BB
Bc                     t          j        d                              dd          } | t           j                            d                              dd| j                  z   }t          | |          }t          | |d	          }|t          j	        |          k    sJ |t          j	        d
          k    sJ |dz  }t          | |          t          j	        d
          k    sJ d S )NF   r   rJ  r   皙ɿr   r   Tr  r]   r,  )
r3   r4   rR   r7   r8   r  r2   r   r   r&  )rK   rI   ndcgndcg_no_tiess       rL   test_ndcg_invariantr    s    Yr]]""1b))Fry,,Q//77c7UUUGfg&&Dfg4@@@L6=......6=%%%%%%tOGfg&&&-*<*<<<<<<<rN   r  c           
      v   dt          j        d          d d         z  }t          j        t          j        ddd          d          }|t           j                            d                              dd	|j        
          z   }t          |||           t          j
        dt          j        t          j        dd                    z            k    sJ t          |||           t          j
        dt          j        t          j        dd                    z            k    sJ t          |||           t          j
        dt          j        t          j        dd                    z            k    sJ t          ||d|           t          j
        dt          j        t          j        dd                    z            k    sJ t          |||           t          j
        dt          j        t          j        dd                    z                                            k    sJ t!          |||           t          j
        dt          j        t          j        dd                    z                                            k    sJ dt          j        d          z  }dt          j        t          j        dd                    z                                  }t          |||           t          j
        |t          j        d          z            k    sJ t          |||           t          j
        t          j        d                    k    sJ t!          |||           t          j
        |          k    sJ t          |||           t          j
        d          k    sJ d S )Nr   r   r   r   rP   )r   r.   r   r  r   r   r  r'   r.   rJ  )log_baser  )r   r   	   r]   )r3   eyetiler4   r7   r8   r  r2   r   r   r&  r  r   log10r   ri  r    r   rS   )r  rK   rI   y_score_noisyexpected_dcg_scores        rL   test_ndcg_toy_examplesr&    s   2A2Fgbi2r**F33Gbi33A66>>c ?   M [  	q2729Q??333	4	45 5 5 5 ;  	q2729Q??333	4	45 5 5 5 [  	q2729Q??333	4	45 5 5 5 "+  	q28BIaOO444	5	56 6 6 6 fg;???6=	
RWRYq!__%%	%++--D D     VW+>>>&-	
RWRYq!__%%	%++--C C      Fbgbi1oo666;;==[  	)BGAJJ6	7	78 8 8 8 [  	rwqzz	"	"# # # # VW+>>>&-C C     fg;???6=QTCUCUUUUUUUrN   c                      t          dd          \  } }| dz   }t          ||           t          j                            d                              d          \  }}t          ||           d S r  )r   _test_ndcg_score_forr3   r7   r8   r  r   s      rL   test_ndcg_scorer)  .  sp    .ALLLIAvgkG)))i++A..<<\JJOFG)))))rN   c                 J   t          | |           }t          | |          }||k                                    sJ | dk                        d          }||          t          j        t	          j        |                                                     k    sJ ||         t          j        t	          j        |                                                    k    sJ ||          t          j        t          | |          |          t          | |           |          z            k    sJ ||         t          j        t	          j        |                                                    k    sJ |j	        | j	        d         fk    sJ |j	        | j	        d         fk    sJ d S )Nr   r.   r  )
r   ru  r   r&  r3   r   rS   r   r   r2   )rK   rI   r
  rb   all_zeros        rL   r(  r(  6  s   //E00EUN!!!!!!  a ((H(v}RWxi__5F5F-G-GHHHHHH?fmBHX\\^^,D,DEEEEEE(v}67++XI6
VV
,
,hY
7	8        ?fmBHX\\^^,D,DEEEEEE;6<?,,,,,;6<?,,,,,,,rN   c            	         t          j        g d          } t          | | d          dk    sJ t          | | d          dk    sJ t          j        t
                    5  t          | | d          sJ 	 d d d            n# 1 swxY w Y   t          j        t
                    5  t          | | d          sJ 	 d d d            n# 1 swxY w Y   t          j        t
                    5  t          | | d          sJ 	 d d d            n# 1 swxY w Y   t          j        g d          }t          | |d          }t          | |          }||k    sJ t          | |d	          d
k    sJ t          d          \  } }}t          j        ddd          D ]1}t          t          | ||          t          | ||                     2d S )Nr  r.   )ru   gMbP?gg?r   )r   r   r   g{Gz?r   r   Tr   g-C6?r   )
r3   r   r   r   r   r   rM   linspacer   r   )rK   r  roc_auc_with_max_fpr_oneunconstrained_roc_aucrJ   rx   ru   s          rL   test_partial_roc_auc_scorer0  F  s   Xlll##F333q88887771<<<<	z	"	" ; ;VVT:::::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;	z	"	" : :VVS9999999: : : : : : : : : : : : : : :	z	"	" 8 8VVQ77777778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 x+++,,H,VXqIII)&(;;#'<<<<<37773>>>>'t444FFA;tQ** 
 
&&':::"667;;	
 	
 	
 	

 
s6    BB	B&CCC,DDDzy_true, k, true_scorer   r'   r   r   c                     t          j        g dg dg dg dg          }t          | ||          }|t          j        |          k    sJ d S )Nr  r   r   r   r   r   r   r   r   r   r   r   r   r  r3   r   r!   r   r&  )rK   r  
true_scorerI   rb   s        rL   test_top_k_accuracy_scorer7  `  su     h            		
 G !A666EFM*--------rN   zy_score, k, true_score)rP   rP   r.   r.   )rP   r.   rP   r.   )r   r   r   r   )r   r   r   r   c                 V   g d}|                                  dk    r|                                 dk    rdnd}|dk    r#| |k                        t          j                  n|}t          || |          }t          ||          }||cxk    rt          j        |          k    sn J d S )Nr  r   r.   r   r  )	minmaxastyper3   int64r!   r   r   r&  )rI   r  r6  rK   rl   rJ   rb   	score_accs           rL    test_top_k_accuracy_score_binaryr>  u  s     \\F{{}}))gkkmmq.@.@aI78Avvg	!))"(3336F A666Evv..II::::z!:!:::::::::rN   zy_true, true_score, labels)r   r.   r.   r'   )r   r.   r.   r.   r.   r.   r.   r.   )r   r=  r=  r   )r   r   r  r=  labels_as_ndarrayc                     |rt          j        |          }t          j        g dg dg dg dg          }t          | |d|          }|t	          j        |          k    sJ dS )z,Test when labels and y_score are multiclass.r  r2  r3  r4  r'   r  r   N)r3   r5  r   r!   r   r&  )rK   r6  r   r@  rI   rb   s         rL   0test_top_k_accuracy_score_multiclass_with_labelsrC    s      $F##h            		
 G !AfEEEEFM*--------rN   c                     t          j        dddd          \  t          d          \  } }}}t          d                              | |           t          | |f||f          D ]O\  fdt          dd          D             }t          j        t          j	        |          dk              sJ Pd S )NrJ  r,  r   )r  rC   n_informativer-   )r-   c                 Z    g | ]'}t                                        |           (S )r  )r!   r=   )r   r  rA   rH   rB   s     rL   r   z8test_top_k_accuracy_score_increasing.<locals>.<listcomp>  sD     
 
 
CD C$5$5a$8$8A>>>
 
 
rN   r'   )
r   make_classificationr#   r$   r<   rY  r_   r3   ru  r   )X_trainX_testy_trainy_testscoresrA   rH   rB   s        @@@rL   $test_top_k_accuracy_score_increasingrM    s    'BQ  DAq (811'M'M'M$GVWf
!
,
,
,CGGGWWf%'899 + +1
 
 
 
 
 
HMaQS
 
 
 vbgfoo)******+ +rN   c                     t          j        g dg dg dg dg          }t          | ||          t          j        |          k    sJ d S )N)r   r   r   r   )r.   r   r   r   )r   r   r   r   r?  r  r5  )rK   r  r6  rI   s       rL   test_top_k_accuracy_score_tiesrO    sh     hLLLLLLLL		
 G  1555z9R9RRRRRRRrN   z	y_true, kr   c                     t          j        g dg dg dg dg          }d}t          j        t          |          5  t          | ||          }d d d            n# 1 swxY w Y   |dk    sJ d S )	Nr  )r   r   r   r   )r   r   r   r   )r   r   r   r   zu'k' \(\d+\) greater than or equal to 'n_classes' \(\d+\) will result in a perfect score and is therefore meaningless.r   r  r.   )r3   r   r   r   r"   r!   )rK   r  rI   r   rb   s        rL   !test_top_k_accuracy_score_warningrQ    s     h            		
 G	6  
,4D	E	E	E ; ;$VW:::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;A::::::s   AA!$A!zy_true, y_score, labels, msg)r   g=
ףp=?r.   r'   )r   r   r   )r   r   r   r   )r   r   r   z9y type must be 'binary' or 'multiclass', got 'continuous'zZNumber of classes in 'y_true' \(4\) not equal to the number of classes in 'y_score' \(3\).)r   r   r   r   z"Parameter 'labels' must be unique.z#Parameter 'labels' must be ordered.)r   r   r.   r'   zSNumber of given labels \(4\) not equal to the number of classes in 'y_score' \(3\).r  z3'y_true' contains labels not in parameter 'labels'.)r   r   r   )r   r   r   z}`y_true` is binary while y_score is 2d with 3 classes. If `y_true` does not contain all the labels, `labels` must be providedc                     t          j        t          |          5  t          | |d|           d d d            d S # 1 swxY w Y   d S )Nr   r'   rB  )r   r   r   r!   )rK   rI   r   r?  s       rL   test_top_k_accuracy_score_errorrS    s    b 
z	-	-	- B BVW&AAAAB B B B B B B B B B B B B B B B B Bs   =AAc                      t          g dg dg          } t          j        g dg dg          }t          | |          }|t	          j        d          k    sJ d S )Nr   r   )r   r   r   r  )r   r3   r   r   r   r&  )rK   rI   results      rL   Otest_label_ranking_avg_precision_score_should_allow_csr_matrix_for_y_true_inputrV  ?  sj     III.//Fh344G267CCFV]5))))))))rN   )NF)r   rD  r   )r   r   numpyr3   rN  scipy.sparser   scipyr   sklearnr   r   sklearn.utils.extmathr   sklearn.datasetsr   sklearn.random_projectionr	   sklearn.utils.validationr
   r   r   sklearn.utils._testingr   r   r   r   sklearn.metricsr   r   r   r   r   r   r   r   r   r   sklearn.metrics._rankingr   r   r   r    r!   sklearn.exceptionsr"   sklearn.model_selectionr#   sklearn.linear_modelr$   sklearn.preprocessingr%   CURVE_FUNCSrM   r[   rf   rn   r   markparametrizer   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r"  r*  r;  r@  rH  rQ  rT  r_  re  ro  rh  rz  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r_   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r&  r)  r(  r0  r7  r>  rC  rM  rO  rQ  rS  rV  r   rN   rL   <module>ri     s   				       # # # # # #                   ) ) ) ) ) ) ; ; ; ; ; ; ; ; ; ; ; ; I I I I I I I I 7 7 7 7 7 7 2 2 2 2 2 2 6 6 6 6 6 6 5 5 5 5 5 5 < < < < < < * * * * * *       3 3 3 3 3 3 * * * * * * % % % % % % A A A A A A 2 2 2 2 2 2 . . . . . . ) ) ) ) ) ) % % % % % % L L L L L L L L 1 1 1 1 1 1 1 1 0 0 0 0 0 0 5 5 5 5 5 5 4 4 4 4 4 4 3 3 3 3 3 3 0 0 0 0 0 0 )# )# )# )#`2 2 2  <  &H H H8 $//
) 
) 0/
)
" 
" 
") ) )&# # #) ) )) ) )8) ) )6eN eN eNPJ J J	) 	) 	). . .&  " 	,,,		+	,,,		&			/			t$	 /X /X /Xd 	,,,		+	&&&	'	'9   : 	,,,		&			t$	yyy!			/	   D      D: : :( 	,hbh|||.D.DiiiP/BH)))**OO	
BH\\\""		
 1BH)))**OO	
BH\\\""F		
BH)))**#J		
BH\\\""LL		
BH)))**   		
 ABH)))**OO	
 ABH)))**OO	
 ABH\\\""II	
i9< <z 77P P 87{< <|P ' "%88	
' "%88	
F "B77	
 "c22	
; E"	
 
4R8K&) )T	0 	0U) )T	0* * *@ {33# # 43# {33 E  E 43 EF {33	& 	& 43	&     4) ) )(` ` `F< < <= = =&$ $ $2 .	KKK!qc*	NNNQC!-	NNNQC!-	%%%sQC0	KKK#,	NNNSEA3/	NNNSEA3/	%%%uqc2	KKK#,	NNNQC!-	NNNQC!-	%%%sQC0	KKKiii8	NNNKKK;	NNNKKK;	%%%{{{III>	KKKMMM:	NNNIII}}}=	NNNIII}}}=	%%%yyy---@) 2' '3 2' .
Q#saS1#&
Q#saS1#&	$$$seaS1	$$$seaS1	$$$qcA3/	$$$qcA3/	$$$qcA3/	$$$qcA3/	 ' ' '   $E$E$EFF* * GF* 																 	 	 	 
Q?@	QH?@	KKK!CD	KKK!CD...OO(	

 " " "> > >48S 8S 8Sv  .1 1 14X X X  6# # #N 9:3 3 3 3@ 3)	  "G!RSS  TS C C C m44j11q22  32 21 54  *4 4 4nL L L  	$$$%	%%%&'
))&&&' ( ( (0 0 0f9 9 9 	P 	P 	P) ) )U U U  (  F F F= = = u66#V #V 76#VL* * *- - - 
 
 
4 	q$	q#	q$ 
. 
. 
. 	...	!	!1a(	...	!	!1c*	...	!	!1a(	&&&	'	'A.	&&&	'	'C0	&&&	'	'A.
 
	; 	;
 
	;  	,,,		|||4	,,,		lll3	,,,		lll3	&&&	'	'/C/C/CD	  ,tUm<<. . =< .&+ + +& 	q$	q#	q! 
S 
S 
S 	q	q   $ " OO	 G
	
 LL	 +	
 !  	 !  0
	
 !  	 OO1
	
 LL	 LL 	
 LL	 IIA
	
 F__ooo.S		
KLO O`B BaO O`B
* * * * *rN   