a
     ldf!                     @   s   d dl Z d dlZd dlZd dlmZ d dlmZ d dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ d dlmZmZ eeZdddZejZG dd dZdS )    N)	getLogger)settings)SoftTimeLimitExceeded)BeautifulSoup)parser)Subquery)timezone)Review)ReviewSourcePlatform)WidgetReivewsWidgetzapplication/jsonz)Mozilla/5.0 (Windows NT 10.0; Win64; x64))zcontent-typez
user-agentc                   @   s6   e Zd ZedddZedddZedddZd	S )
HomeAdvisorCrawler)urlc                 C   s*   t tj|tdjd}t|dj S )Nheaderszhtml.parsera  #app > div > div.ha_sp-profile-body-container.\@px-2.md\:\@px-6.\@pb-5.\@bg-gray-300.\@h-auto.\@flex.\@flex-col.\@items-center.\@w-full > main > div.ha_ratings.\@mt-4 > div > div > section.sp--rating_overall.short-border-r.\@border-none.\@flex-grow.\@flex-shrink.\@pt-4.\@pl-0.\@pr-12 > div > div.numeric-rating.\@font-general.\@font-bold.md\:\@text-xl.\@text-base.\@border-2.\@border-solid.\@border-gray-350.md\:\@text-gold.\@text-gold.\@ml-2.md\:\@ml-0)r   requestsgetr   textfloat
select_onestrip)selfr   soup r   K/var/www/html/Ranjet/backend/verify_trusted/crawlers/homeadvisor/crawler.pyget_avg_rating   s    z!HomeAdvisorCrawler.get_avg_ratingc              
      s.  t d z| |}g }|dd }d}d}d}||krd| d| d}t| tj|td	}t	|j
}	|dkr|	d
 d }|	d
 d }t | ttj|	dd |	d  | fdd D 7 }|d7 }t|tkr2|d t }qq2|||fW S  ty( }
 zt |
 W Y d }
~
d S d }
~
0 0 d S )Nzn==================================================HomeAdvisor Crawler=========================================.   r   z'https://www.homeadvisor.com/sm/reviews/z?page=z&sort=newest&pageSize=10r   
paginationZtotalResultCountZ	pageCount   )indentratingsc              	      sF   g | ]>}|d  d |d t j  d d d d|d |d d	qS )
ZspRatingCommentZratingIdZconsumerNamer   Z
createDateg     @@ commentZoverallRating)idauthordateheadlinebodyrating)datetimefromtimestamp).0reviewreviewsr   r   
<listcomp>2   s   	
z2HomeAdvisorCrawler.get_reviews.<locals>.<listcomp>)printr   splitloggerdebugr   r   r   jsonloadsr   dumpslenmax_crawler	Exception)r   r   
avg_ratingparsed_reviews
company_idcount
total_pagetotal_reviewsrjson_responseer   r0   r   get_reviews   s<    

	zHomeAdvisorCrawler.get_reviewsreview_sourcec              
      s  |   j}|d u r*tjj _   d S |\}}} fdd|D }tjj	 j
ddjttjjddddd  tjj|dd	 ztjj	 j
ttjjddddd
jddd}tjj	|d}|D ]X}	tjj	 j
|	j|	jdjt|d}z|d j}
|	j|
d W n   Y n0 |  qtjj	ttjj	 jdjdddddddd}t|}|dk rtjj	ttjj	 jdjddddddddd d|  }tjj	|djdd n^|dkr tjj	ttjj	 jdjdddddddddd  }tjj	|djdd d |du rtjj	 jd }tjj	|j
tjjdjddd}t|}g }|dk rtjj	ttjj	 jtjjdjdddddddd d|  }|D ]}|t||d qtjj|dd	 n
|dkrtjj _t t!j!"  d | d!|  | _#t t!j!"  d" | _$t t!j!"  d#    t d$ W n@ t%y } z&tjj _   t | W Y d }~n
d }~0 0 d S )%Nc                    s@   g | ]8}t  |d  |d |d |d |d t |d dqS )r'   r(   r)   r*   r+   r&   )sourcer'   r(   r)   r*   r+   
date_parseexternal_id)r	   r   now)r.   rC   rG   r   r   r2   N   s   z3HomeAdvisorCrawler.sync_reviews.<locals>.<listcomp>T)	source_id	lock_edit	review_id)flatF)id__indisplay_order__isnull)ignore_conflicts)rM   rQ   rR   r&   )rQ   )rM   r*   r(   r   )rK   )r?   )source_id__in	is_activedisplay_orderz-date_modifyz-date   )pk__in)rU   )rU   rV   )company)widget review__source__platform__statusr/      )r?   platform__status)rT   )rZ   r/   z - z : z - set total reviewsz - set avg ratingdone)&rF   r   r
   
SyncStatusFALSEsync_statussaver	   objectsfilterr&   excluder   r   values_listdeletebulk_creater*   r(   listrK   updater?   order_byr:   r   firstr   StatusACTIVEappendSYNCEDr3   r,   rL   reviews_countaverage_ratingr   )r   rH   is_addresultrB   r=   r1   dup_reviews_idsuncheck_reviewsrowrK   active_reviewsnum_active_reviewsnested_qrZ   widget_reviewsnum_widget_reviewswsnested_wwrE   r   rG   r   sync_reviewsG   s    









 





zHomeAdvisorCrawler.sync_reviewsN)__name__
__module____qualname__strr   rF   r
   r   r   r   r   r   r      s   )r   ) r,   r7   timeloggingr   django.confr   r   Zbilliard.exceptionsr   bs4r   dateutilr   django.db.modelsr   django.utilsr   verify_trusted.companies.modelsr	   verify_trusted.reviews.modelsr
   r   Zverify_trusted.widgets.modelsr   r   r   r5   r   MAX_CRAWLER_SIZEr;   r   r   r   r   r   <module>   s&   