a
     ld                      @   s   d dl Z d dlZd dlZd dlmZ d dlmZ d dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZmZ eeZejZG dd dZdS )    N)	getLogger)settings)SoftTimeLimitExceeded)parser)Subquery)timezone)Review)ReviewSourcePlatform)WidgetReivewsWidgetc                   @   s(   e Zd ZedddZedddZdS )CheckatradeCrawler)urlc              
      s  d}d}z^t d |dd }zFtd|  }|d }|d }ttd| d	 d
 }W n   d}Y n0 d}t | g  ||kr^d| d| d}t | t|}	|	 d }
t tj|
dd  dd |
D 7  tt	  t   |d7 }t	 t
kr fddt D   d t
  |dkr^|dkr^d }d }q^q|| fW S  ty } zt| W Y d }~d S d }~0 0 d S )Nr   g        zd====================================checkatrade crawler=============================================/z0https://trade-profile.checkatrade.com/companies/reviews_countratingz6/reviews?page=1&itemsPerPage=18&sortBy=createDate.descpages   z/reviews?page=z'&itemsPerPage=18&sortBy=createDate.descitems   )indentc              
   S   sf   g | ]^}|d  dur|d dt |d  t|d p4d t|d pFd t|d d dd	qS )
dateNidReviewertitle bodyr   r   )r   authorr   headliner   r   )r   parsestrstripround).0review r&   K/var/www/html/Ranjet/backend/verify_trusted/crawlers/checkatrade/crawler.py
<listcomp>1   s   	z2CheckatradeCrawler.get_reviews.<locals>.<listcomp>c                    s(   g | ] \}}| |d  d vr|qS )r   Nr&   )r$   jkparsed_reviewsr&   r'   r(   A       )loggerdebugsplitrequestsgetjsonintdumpsprintlenmax_crawler	enumerate	Exception)selfr   total_reviews
avg_ratingcompany_nameZgeneral_contentZpage_numnum	crawl_urlresreviewser&   r+   r'   get_reviews   sX    






	
zCheckatradeCrawler.get_reviewsreview_sourcec              
      s  |   j\}}}|d u r0tjj _   d S  fdd|D }tjj	 j
ddjttjjddddd  tjj|dd	 ztjj	 j
ttjjddddd
jddd}tjj	|d}|D ]X}tjj	 j
|j|jdjt|d}z|d j}	|j|	d W n   Y n0 |  qtjj	ttjj	 jdjdddddddd}
t|
}|dk rtjj	ttjj	 jdjddddddddd d|  }tjj	|djdd n^|dkrtjj	ttjj	 jdjdddddddddd  }tjj	|djdd d |du rtjj	 jd }tjj	|j
tjjdjddd}t|}g }|dk rtjj	ttjj	 jtjjdjdddddddd d|  }|D ]}|t||d qtjj|dd	 n
|dkrtjj _t t!j!"  d | d!|  | _#t t!j!"  d" t$|d# d# _%t t!j!"  d$    t d% W n@ t&y } z&tjj _   t | W Y d }~n
d }~0 0 d S )&Nc                    s@   g | ]8}t  |d  |d |d |d |d t |d dqS )r   r   r   r   r   r   )sourcer   r   r   r   r   
date_parseexternal_id)r   r   now)r$   rrE   r&   r'   r(   R   s   z3CheckatradeCrawler.sync_reviews.<locals>.<listcomp>T)	source_id	lock_edit	review_id)flatF)id__indisplay_order__isnull)ignore_conflicts)rL   rP   rQ   r   )rP   )rL   r   r   r   )rI   )
company_id)source_id__in	is_activedisplay_orderz-date_modifyz-date   )pk__in)rU   )rU   rV   )company)widget review__source__platform__statusr%      )rS   platform__status)rT   )rZ   r%   z - z : z - set total reviewsr   z - set avg ratingdone)'rD   r   r	   
SyncStatusFALSEsync_statussaver   objectsfilterr   excluder   r   values_listdeletebulk_creater   r   listrI   updaterS   order_byr7   r   firstr
   StatusACTIVEappendSYNCEDr6   datetimerJ   r   r#   average_ratingr   )r;   rF   is_addr<   r=   rB   dup_reviews_idsuncheck_reviewsrowrI   active_reviewsnum_active_reviewsnested_qrZ   widget_reviewsnum_widget_reviewswsnested_wwrC   r&   rE   r'   sync_reviewsL   s    








 





zCheckatradeCrawler.sync_reviewsN)__name__
__module____qualname__r!   rD   r	   r   r&   r&   r&   r'   r      s   5r   )rq   r3   timeloggingr   django.confr   r1   Zbilliard.exceptionsr   dateutilr   django.db.modelsr   django.utilsr   verify_trusted.companies.modelsr   verify_trusted.reviews.modelsr	   r
   Zverify_trusted.widgets.modelsr   r   r   r.   MAX_CRAWLER_SIZEr8   r   r&   r&   r&   r'   <module>   s   