a
     ld#                     @   s   d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ d dlmZmZ eeZdddZejZ G dd dZ!dS )    N)	getLogger)Tuple)settings)SoftTimeLimitExceeded)BeautifulSoup)parser)Subquery)timezone)Review)ReviewSourcePlatform)WidgetReivewsWidgetzapplication/jsonz)Mozilla/5.0 (Windows NT 10.0; Win64; x64))zcontent-typez
user-agentc                   @   s0   e Zd Zdd ZedddZedddZd	S )

BBBCrawlerc           	   
   C   s  t tj|tdjd}z2t|dj dddd}t	d| W n2 t
yz } zt	d| d	}W Y d }~n
d }~0 0 z2t|d
jdddd }t	d| W n2 t
y } zt	d| d}W Y d }~n
d }~0 0 |dd }|dd }|dd }||||fS )Nheaderszhtml.parserzudiv.MuiPaper-root.MuiPaper-elevation.MuiPaper-rounded.MuiPaper-elevation1.MuiCard-root.css-1dcbtqe > div > div > spanz/5 stars
avg_ratingzavg_rating Eg      ?zldiv.MuiPaper-root.MuiPaper-elevation.MuiPaper-rounded.MuiPaper-elevation1.MuiCard-root.css-1dcbtqe > div > pzCustomer Reviewsz
Average ofZtotal_reviewztotal_reviews E   /-)r   requestsgetr   textfloat
select_onestripreplaceprint	Exceptionintsplit)	selfurlsoupr   Etotal_reviewsZ
id_companyZbbb_idbussiness_id r+   C/var/www/html/Ranjet/backend/verify_trusted/crawlers/bbb/crawler.pyget_info   sD    


zBBBCrawler.get_info)r&   c              
   C   s  zH|  |d \}}}}t| d| d| d|  g }d}d}t||k sZ||kr@t| zd| d| d| d}	t|	 tj|	td}
t	|
j
}|dkr|d	 }ttj|d
d |d }|dd |D 7 }t|tkr|d t }W q@|d7 }W qD ty< } zt| |d7 }W Y d }~qDd }~0 0 qD|||fW S  tyz } zt| W Y d }~d S d }~0 0 d S )Nz/customer-reviewsz, r   z=https://www.bbb.org/api/businessprofile/customerreviews?page=z&pageSize=10&businessId=z&bbbId=z$&sort=reviewDate%20desc,%20id%20descr   Z
totalPages   )indentitemsc              
   S   sv   g | ]n}|d  |d t jt|d d t|d d t|d d dd|d r`|d	 d
 d n|d |d dqS )iddisplayNamedateyearmonthday)r4   r5   r6   r   ZhasExtendedTextZextendedTextr   r   ZreviewStarRating)r1   authorr3   headlinebodyrating)datetimer3   r#   ).0reviewr+   r+   r,   
<listcomp>G   s   z*BBBCrawler.get_reviews.<locals>.<listcomp>)r-   r!   lenloggerdebugr   r   r   jsonloadsr   dumpsmax_crawlerr"   )r%   r&   r)   r   ZbbbIDr*   parsed_reviewspageZ
total_page	crawl_urlrZjson_responsereviewser(   r+   r+   r,   get_reviews5   s>    
 zBBBCrawler.get_reviews)review_sourcec                 C   s*  z|  |j\}}}W n> tyT } z&tjj|_|  t| W Y d }~n
d }~0 0 t| d| dt	|  g }|D ]>}|
t||d |d |d |d |d t |d d	 qztt	|  tjj|jd
djttjjdd
ddd  tjj|dd ztjj|jttjjdd
dddjdd
d}	tjj|	d}
|
D ]Z}tjj|j|j|jdjt|	d}z|d j}|j|d W n   Y n0 |  qLtjjttjj|jdjdd
dd
dddd}t	|}|dk rFtjjttjj|jdjdd
ddddddd d|  }tjj|djd
d n^|dkrtjjttjj|jdjdd
dd
dddddd  }tjj|djdd d |d
u rvtjj|jd }tjj|jt j!j"d jd!d
d}t	|}g }|d"k rltjjttjj|jt j!j"d#jdd
dd$dddd d"|  }|D ]}|
t||d% q@tjj|dd n
|d"krvtjj#|_tt$j$  d&| d'|  ||_%tt$j$  d( ||_&tt$j$  d) |  td* W n@ ty$ } z&tjj|_|  t| W Y d }~n
d }~0 0 d S )+Nz: z; r7   r3   r8   r9   r:   r1   )sourcer7   r3   r8   r9   r:   
date_parseexternal_idT)	source_id	lock_edit	review_id)flatF)id__indisplay_order__isnull)ignore_conflicts)rQ   rU   rV   )rU   )rQ   r9   r3   r   )rP   )
company_id)source_id__in	is_activedisplay_orderz-date_modifyz-date   )pk__in)rZ   )rZ   r[   )company)widget review__source__platform__statusr=      )rX   platform__status)rY   )r_   r=   z - z : z - set total reviewsz - set avg ratingdone)'rL   r&   r"   r   
SyncStatusFALSEsync_statussaver!   r?   appendr
   r	   nowobjectsfilterr1   excluder   r   values_listdeletebulk_creater9   r3   listrP   updaterX   order_byr   firstr   StatusACTIVESYNCEDr;   reviews_countaverage_rating)r%   rM   is_addr)   r   Zraw_reviewsrK   Zformated_reviewsrI   dup_reviews_idsuncheck_reviewsrowrJ   rP   active_reviewsnum_active_reviewsnested_qr_   widget_reviewsnum_widget_reviewswsnested_wwr+   r+   r,   sync_reviews`   s    






 





zBBBCrawler.sync_reviewsN)__name__
__module____qualname__r-   strrL   r   r   r+   r+   r+   r,   r      s   +r   )"r;   rB   timeloggingr   typingr   django.confr   r   Zbilliard.exceptionsr   bs4r   dateutilr   django.db.modelsr   django.utilsr	   verify_trusted.companies.modelsr
   verify_trusted.reviews.modelsr   r   Zverify_trusted.widgets.modelsr   r   r   r@   r   MAX_CRAWLER_SIZErE   r   r+   r+   r+   r,   <module>   s(   