o
    /kjâG  ã                   @   s   d dl Z d dlZd dlmZ d dlmZ d dlmZmZm	Z	m
Z
 d dlZd dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d
ZdZdZdZh d£ZdZeG dd„ dƒƒZdedefdd„Z dVdedededefdd„Z!de"defdd„Z#dWdede$defd d!„Z%d"ed#ed$edefd%d&„Z&d'e$d(ed#ed$edeee$f f
d)d*„Z'd+d,„ Z(d-e	e d.e	e de"fd/d0„Z)d-e	e d.e	e de"fd1d2„Z*d3eeef defd4d5„Z+d6e	e de,fd7d8„Z-d9e j.de	eeef  fd:d;„Z/d"ede	eeef  fd<d=„Z0d9e j.de	eeef  fd>d?„Z1d9e j2de	eeef  fd@dA„Z3dBeeef dCe4de
e fdDdE„Z5dedefdFdG„Z6dHedIe$deeef fdJdK„Z7d'e$dLe
e" dMedNe
e" d#edeee$f fdOdP„Z8d#ed$ede$fdQdR„Z9dIe$dSe"d#edeee$f fdTdU„Z:dS )Xé    N)Ú	dataclass)ÚPath)ÚCallableÚDictÚListÚOptional)ÚTfidfVectorizer)Úaccuracy_score)Úcross_val_scoreÚtrain_test_split)ÚComplementNB)Útext)ÚlabelÚcategoryÚclassÚtargetÚv1Ú
predictionÚpredictions)r   ÚmessageÚbodyÚemailÚcontentÚmailÚv2)ÚsubjectÚtitle)ÚsenderÚsender_emailÚfromÚ
from_email>   ÚidÚemail_idr   r   úemail noú	email no.g¸…ëQ¸î?c                   @   sF   e Zd ZU eed< eed< eed< eed< eed< eed< eed< dS )	ÚUploadTrainingResultÚ
total_rowsÚtraining_rowsÚreclassified_rowsÚaccuracyÚ
spam_countÚ	ham_countÚundecided_countN)Ú__name__Ú
__module__Ú__qualname__ÚintÚ__annotations__Úfloat© r3   r3   ú$/home/esfera/manjet/spam_training.pyr%      s   
 r%   ÚvalueÚreturnc                 C   s:   t | tƒsdS |  ¡ } t dd| ¡} t dd| ¡ ¡ } | S )NÚ z[^a-z0-9\s]ú z\s+)Ú
isinstanceÚstrÚlowerÚreÚsubÚstrip©r5   r3   r3   r4   Ú
clean_text)   s   
r@   r   r   r   c                 C   s>   g }||| fD ]}t |tƒr| ¡ r| |¡ qtd |¡ƒS )Nr8   )r9   r:   r>   Úappendr@   Újoin)r   r   r   Úpartsr5   r3   r3   r4   Úemail_learning_text2   s   
€rD   Úscorec                 C   s   | dkrdS | dkrdS dS )NgÍÌÌÌÌÌä?Úspamgš™™™™™Ù?ÚhamÚ	undecidedr3   )rE   r3   r3   r4   Úclassify_probability:   s
   rI   éÈ   Úmax_lenc                 C   s<   | pd  dd¡  dd¡ ¡ } t| ƒ|kr| d |… d S | S )Nr7   Ú
r8   úz...)Úreplacer>   Úlen)r   rK   r3   r3   r4   Úget_email_summaryB   s    rP   Úcsv_pathÚsession_factoryÚ	model_dirc              	   C   sô   t | ƒ}dd„ |D ƒ}t|ƒdk rtdƒ‚dd„ |D ƒ}dd„ |D ƒ}tt|ƒƒdk r/tdƒ‚tƒ \}}t||||ƒ}	t|jƒ d¡}
|j	d	d	d
� t
 ||d ¡ t
 ||d ¡ t|||
|	|d�}tt|ƒt|ƒ|d |	|d |d |d d�S )Nc                 S   s0   g | ]}|d  dv rt |ƒrt |ƒ|d  f‘qS )r   >   rG   rF   )Ú_build_row_text©Ú.0Úrowr3   r3   r4   Ú
<listcomp>M   s
    þz-process_uploaded_spam_csv.<locals>.<listcomp>é   zOCSV must contain at least two labeled rows with spam/ham labels and email text.c                 S   ó   g | ]}|d  ‘qS )r   r3   rU   r3   r3   r4   rX   V   ó    c                 S   rZ   )é   r3   rU   r3   r3   r4   rX   W   r[   zCCSV must contain both spam and ham examples so the model can train.rF   T)ÚparentsÚexist_okúfinal_model.pklúfinal_vectorizer.pkl)ÚmodelÚ
vectorizerÚ
spam_indexr)   rR   ÚtotalrG   rH   )r&   r'   r(   r)   r*   r+   r,   )Ú_read_email_datasetrO   Ú
ValueErrorÚsetÚ_create_trainerÚ_train_and_score_modelÚlistÚclasses_ÚindexÚmkdirÚjoblibÚdumpÚ_reclassify_existing_emailsr%   )rQ   rR   rS   Úrowsr'   ÚtextsÚlabelsrb   ra   r)   rc   Úreclassificationr3   r3   r4   Úprocess_uploaded_spam_csvG   s@   þ
ûùru   r"   Úfeedback_labelc                 C   s*  |pd  ¡  ¡ }|dvrtdƒ‚|d }|d }| ¡ r | ¡ s$tdƒ‚|ƒ }z| tdƒd| i¡ ¡  ¡ }W | ¡  n| ¡  w |sGtd	ƒ‚t	|d
 |d ƒ}|sVtdƒ‚t
 |¡}	t
 |¡}
t|	jƒ d¡}|
 |g¡}|	j||gddgd� t
 |	|¡ t||	|
|ƒ}t| |d |d |d |d�S )Nr7   >   rG   rF   z2Model learning only supports spam or ham feedback.r_   r`   z<Upload and train a dataset before submitting model feedback.z¼
                SELECT id, subject, body, model_accuracy
                FROM email_filter
                WHERE id = :email_id
                  AND imap_uid IS NOT NULL
                r"   z8Could not find an IMAP email body for feedback learning.r   r   zAThis email has no usable subject or body text for model learning.rF   rG   )ÚclassesÚspam_probabilityÚpredicted_labelÚmodel_accuracy)r"   rx   ry   rz   rR   )r>   r;   rf   ÚexistsÚexecuter   ÚmappingsÚfirstÚcloserD   rn   Úloadrj   rk   rl   Ú	transformÚpartial_fitro   Ú_predict_emailÚ_update_email_prediction)r"   rv   rR   rS   Ú
model_pathÚvectorizer_pathÚdbrW   Úcleanedra   rb   rc   Úfeaturesr   r3   r3   r4   Úlearn_from_feedbackw   sJ   ÿ÷
ö

ûrŠ   c                  C   s$   t dddddd�} tdd�}| |fS )	N)r\   rY   r\   iø$ Tz(?u)\b\w+\b)Úngram_rangeÚmin_dfÚmax_featuresÚsublinear_tfÚtoken_patternçš™™™™™¹?©Úalpha)r   r   )rb   ra   r3   r3   r4   rh   °   s   û
rh   rr   rs   c                 C   s*   t ||ƒ}| |¡}|  ||¡ t|tƒS ©N)Ú_estimate_accuracyÚfit_transformÚfitÚmaxÚMIN_REPORTED_ACCURACY)ra   rb   rr   rs   Úmeasuredr‰   r3   r3   r4   ri   ¼   s   


ri   c           
      C   s¬   t ƒ \}}| | ¡}| ||¡ tt|| |¡ƒƒ}t| ƒdko,t| d¡| d¡ƒdk}|s1|S tdt| d¡| d¡ƒƒ}t	t
dd�|||dd	�}t| ¡ ƒ}	t||	ƒS )
Né   rF   rG   rY   é   r�   r‘   r)   )ÚcvÚscoring)rh   r•   r–   r2   r	   ÚpredictrO   ÚminÚcountr
   r   Úmeanr—   )
rr   rs   Útrain_vectorizerÚtrain_modelr‰   Útrain_accuracyÚcan_evaluateÚfoldsÚ	cv_scoresÚcv_accuracyr3   r3   r4   r”   Ã   s"   

&û
r”   rW   c                 C   s   t |  d¡|  d¡|  d¡ƒS )Nr   r   r   )rD   Úget)rW   r3   r3   r4   rT   Ù   s   rT   Ú
fieldnamesc                 C   sz   dd„ | D ƒ}d|vrd|vrdS d|v sd|v rdS d	}| D ]}|  ¡  ¡ }|tv s3|tv s3|tv r4q|d
7 }q|dkS )Nc                 S   s   h | ]
}|r|  ¡  ¡ ’qS r3   ©r>   r;   ©rV   Úfieldr3   r3   r4   Ú	<setcomp>Þ   s    z+_is_bag_of_words_dataset.<locals>.<setcomp>r   r   Fr$   r#   Tr   r\   é2   )r>   r;   ÚBOW_META_COLUMNSÚLABEL_COLUMNSÚTEXT_COLUMNS)rª   Ú
normalizedÚnumeric_liker­   Únamer3   r3   r4   Ú_is_bag_of_words_datasetÝ   s   
r¶   Úreaderc              
      s  | j pg }dd„ |D ƒ‰ tˆ dƒ}|stdƒ‚‡ fdd„ˆ D ƒ}g }| D ]\}t| |¡ƒ}|dvr2q$g }|D ]7}| |¡p>d ¡ }	|	sDq6ztt|	ƒƒ}
W n	 tyU   Y q6w |
d	kr[q6t|ƒ}|sbq6| 	|gt
|
d
ƒ ¡ q6d |¡}|svq$| ||dddœ¡ q$|s‡tdƒ‚|S )Nc                 S   ó   i | ]}|r|  ¡  ¡ |“qS r3   r«   r¬   r3   r3   r4   Ú
<dictcomp>î   ó    z)_rows_from_bow_reader.<locals>.<dictcomp>)r   r   z+Bag-of-words CSV needs a Prediction column.c                    s   g | ]
}|t vrˆ | ‘qS r3   )r°   )rV   rµ   ©Únormalized_fieldsr3   r4   rX   ó   s
    þz)_rows_from_bow_reader.<locals>.<listcomp>>   rG   rF   r7   r   r›   r8   ©r   r   r   r   z5Bag-of-words CSV did not contain usable labeled rows.)rª   Ú_find_columnrf   Ú_normalize_labelr©   r>   r0   r2   r@   ÚextendrŸ   rB   rA   )r·   rª   Ú	label_colÚfeature_columnsÚparsed_rowsrW   r   ÚwordsÚcolumnÚ	raw_valuer    ÚtokenÚ
text_valuer3   r»   r4   Ú_rows_from_bow_readerì   sT   


þÿ
üÿ	rÉ   c              	   C   st  | j dddd��¨}| d¡}| d¡ | ¡ stdƒ‚zt ¡ j|dd	�}W n tjy3   tj	}Y nw z	t ¡  
|¡}W n tjyI   d
}Y nw tj||d�}t|g ƒ}| d¡ dd„ |D ƒ}t|ttƒ@ ƒoqt|ttƒ@ ƒ}|pu|}	|	ržtj||d�}
|
jpƒg }t|ƒr“t|
ƒW  d   ƒ S t|
ƒW  d   ƒ S tj||d�}
t|
ƒW  d   ƒ S 1 s³w   Y  d S )NÚrz	utf-8-sigr7   )ÚencodingÚnewlinei   r   zCSV file is empty.z,	;|)Ú
delimitersF)Údialectc                 S   s   h | ]}|  ¡  ¡ ’qS r3   r«   )rV   r5   r3   r3   r4   r®   2  s    z&_read_email_dataset.<locals>.<setcomp>)ÚopenÚreadÚseekr>   rf   ÚcsvÚSnifferÚsniffÚErrorÚexcelÚ
has_headerr·   ÚnextÚboolrg   r±   r²   Ú
DictReaderrª   r¶   rÉ   Ú_rows_from_dict_readerÚ_rows_from_plain_reader)rQ   Úcsv_fileÚsamplerÎ   Úsniffed_headerÚfirst_row_readerÚ	first_rowÚnormalized_first_rowÚhas_named_headerr×   r·   rª   r3   r3   r4   re      sD   


ÿÿ


ÿ
åä$áre   c                 C   sà   | j pg }dd„ |D ƒ}t|tƒ}t|tƒ}t|tƒ}t|tƒ}|r$|s(tdƒ‚g }| D ];}| |¡p4d ¡ }	|rB| |¡p?d ¡ nd}
|rO| |¡pLd ¡ nd}t	|	|
|ƒsXq,| 
t| |¡ƒ|	|
|dœ¡ q,|sntdƒ‚|S )Nc                 S   r¸   r3   r«   r¬   r3   r3   r4   r¹   E  rº   z*_rows_from_dict_reader.<locals>.<dictcomp>z8CSV needs a label column and a text/message/body column.r7   r½   ú*CSV did not contain any usable email rows.)rª   r¾   r±   r²   ÚSUBJECT_COLUMNSÚSENDER_COLUMNSrf   r©   r>   rD   rA   r¿   )r·   rª   r¼   rÁ   Útext_colÚsubject_colÚ
sender_colrÃ   rW   rÈ   Úsubject_valueÚsender_valuer3   r3   r4   rÛ   C  s2   




üÿ	rÛ   c                 C   sZ   g }| D ] }t |ƒdk rq|d  ¡ }|sq| t|d ƒ|dddœ¡ q|s+tdƒ‚|S )NrY   r\   r   r7   r½   rä   )rO   r>   rA   r¿   rf   )r·   rÃ   rW   rÈ   r3   r3   r4   rÜ   d  s"   
üÿ	rÜ   r¼   Ú
candidatesc                 C   s"   |D ]}|| v r| |   S qd S r“   r3   )r¼   rì   Ú	candidater3   r3   r4   r¾   z  s
   ÿr¾   c                 C   s0   t | pdƒ ¡  ¡ } | dv rdS | dv rdS | S )Nr7   >   Ú1ÚyesÚjunkrF   ÚtruerF   >   únot spamÚ0ÚnorG   ÚfalseÚnot_spamrG   )r:   r>   r;   r?   r3   r3   r4   r¿   �  s   r¿   Úcleaned_bodyrc   c                 C   s>   | sd ddœS |  | g¡}t| |¡d | ƒ}|t|ƒdœS )NrH   )rx   ry   r   )r�   r2   Úpredict_probarI   )r÷   ra   rb   rc   r‰   rx   r3   r3   r4   rƒ   Š  s   
þrƒ   rx   ry   rz   c                 C   sˆ   |ƒ }z.z|  tdƒ| |||dœ¡ ¡ }|stdƒ‚| ¡  W n ty+   | ¡  ‚ w W | ¡  n| ¡  w dddddœ}d||< |S )NaI  
                UPDATE email_filter
                SET spam_probability = :spam_probability,
                    predicted_label = :predicted_label,
                    model_accuracy = :model_accuracy
                WHERE id = :email_id
                  AND imap_uid IS NOT NULL
                RETURNING id
                )r"   rx   ry   rz   z-Could not update model scores for this email.r   r\   )rF   rG   rH   rd   )r|   r   r~   rf   ÚcommitÚ	ExceptionÚrollbackr   )r"   rx   ry   rz   rR   r‡   ÚupdatedÚcountsr3   r3   r4   r„   –  s4   ÿüôîþÿr„   c                 C   sd  |d }|d }|  ¡ r|  ¡ sdS | ƒ }z-| tdƒ¡ ¡ }|r'|d d u r.W | ¡  dS |d }| tdƒ¡ ¡  ¡ }W | ¡  n| ¡  w |sLdS t |¡}t |¡}	t	|j
ƒ d¡}
g }|D ]"}t|d |d ƒ}t|||	|
ƒ}| |d	 |d
 |d |dœ¡ qb| ƒ }z$z| tdƒ|¡ | ¡  W n ty£   | ¡  ‚ w W | ¡  t|ƒS | ¡  w )Nr_   r`   r   zî
                SELECT model_accuracy
                FROM email_filter
                WHERE imap_uid IS NOT NULL
                  AND model_accuracy IS NOT NULL
                ORDER BY id DESC
                LIMIT 1
                zµ
                SELECT id, subject, body
                FROM email_filter
                WHERE imap_uid IS NOT NULL
                  AND predicted_label IS NULL
                rF   r   r   r!   rx   ry   ©r!   rx   ry   rz   zû
                UPDATE email_filter
                SET spam_probability = :spam_probability,
                    predicted_label = :predicted_label,
                    model_accuracy = :model_accuracy
                WHERE id = :id
                )r{   r|   r   r~   r   r}   Úallrn   r€   rj   rk   rl   rD   rƒ   rA   rù   rú   rû   rO   )rR   rS   r…   r†   r‡   Úaccuracy_rowr)   Úpending_rowsra   rb   rc   ÚupdatesrW   rˆ   r   r3   r3   r4   Úclassify_new_imap_emailsÀ  st   ÿÿõóÿÿ	÷

üÿ	ÿ	öþÿ
þr  r)   c                 C   sì   |ƒ }zmz`|  tdƒ¡ ¡  ¡ }g }ddddœ}|D ].}	t|	d |	d ƒ}
t|
| ||ƒ}|d }|d }||  d7  < | |	d	 |||d
œ¡ q|rS|  tdƒ|¡ | ¡  t|ƒ|d< |W W | 	¡  S  t
yp   | ¡  ‚ w | 	¡  w )Nz‡
                SELECT id, subject, body
                FROM email_filter
                WHERE imap_uid IS NOT NULL
                r   )rF   rG   rH   r   r   ry   rx   r\   r!   rþ   a  
                    UPDATE email_filter
                    SET spam_probability = :spam_probability,
                        predicted_label = :predicted_label,
                        model_accuracy = :model_accuracy
                    WHERE id = :id
                    rd   )r|   r   r}   rÿ   rD   rƒ   rA   rù   rO   r   rú   rû   )ra   rb   rc   r)   rR   r‡   Úexisting_rowsr  rý   rW   rˆ   r   ry   rx   r3   r3   r4   rp     sP   ÿÿø
üÿ	ÿ	ö
üþ
rp   )NN)rJ   );rÒ   r<   Údataclassesr   Úpathlibr   Útypingr   r   r   r   rn   Úsklearn.feature_extraction.textr   Úsklearn.metricsr	   Úsklearn.model_selectionr
   r   Úsklearn.naive_bayesr   Ú
sqlalchemyr   r±   r²   rå   ræ   r°   r˜   r%   Úobjectr:   r@   rD   r2   rI   r0   rP   ru   rŠ   rh   ri   r”   rT   rÙ   r¶   rÚ   rÉ   re   rÛ   r·   rÜ   Útupler¾   r¿   rƒ   r„   r  rp   r3   r3   r3   r4   Ú<module>   sœ    
	ÿþý
ü0ÿþýü

û9 4 # !"	ÿþýüû

ú*Qýüû
ú