a
    ŠxdÁ  ã                   @   s4  d dl Z d dlZd dlmZ d dlZd dlZd dlmZ d dlm	Z	m
Z
 d dlZd dlZd dlmZmZmZ d dlmZ d dlZd dlmZ d dlmZ d	d
lmZ d	dlmZmZmZmZmZ d	dlm Z m!Z!m"Z"m#Z#m$Z$m%Z% d	dl&m'Z'm(Z( d	dl)m*Z*m+Z+m,Z,m-Z-m.Z. d	dlm/Z/ e 0e1¡Z2G dd„ dƒZ3dS )é    N)Úurljoin)Úpartial)ÚIteratorÚUnion)Úparse_qsÚurlparseÚunquote)Údatetime)ÚRequestException)ÚHTMLSessioné   )Úutils)ÚDEFAULT_PAGE_LIMITÚFB_BASE_URLÚFB_MOBILE_BASE_URLÚFB_W3_BASE_URLÚFB_MBASIC_BASE_URL)Úextract_group_postÚextract_postÚextract_photo_postÚextract_story_postÚPostExtractorÚextract_hashtag_post)ÚPostÚProfile)Úiter_group_pagesÚ
iter_pagesÚiter_photosÚiter_search_pagesÚiter_hashtag_pages)Ú
exceptionsc                   @   s¢  e Zd ZdZeZddddœZdZdAdd	„Zd
d„ Z	dd„ Z
dBdd„Zeee dœdd„Zeee dœdd„Zeee dœdd„Zeee dœdd„Zi dfee dœdd„Zeee dœd d!„Zee dœd"d#„ZdCee dœd$d%„Zedœd&d'„Zee dœd(d)„Zedœd*d+„Zedœd,d-„Zee dœd.d/„Zeeef ee d0œd1d2„Z d3d4„ Z!d5d6„ Z"i fd7d8„Z#eed9œd:d;„Z$e%dœd<d=„Z&e'dddd>fd?d@„Z(dS )DÚFacebookScraperz,Class for creating FacebookScraper Iteratorszen-US,en;q=0.5z?1úuMozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8)zAccept-LanguagezSec-Fetch-Userú
User-AgentFNc                 C   sD   |d u rt ƒ }|j | j¡ |d u r(i }|| _ddi|d< || _d S )NÚlocaleÚen_USÚparams)r   ÚheadersÚupdateÚdefault_headersÚsessionÚrequests_kwargs)Úselfr*   r+   © r-   úY/var/www/html/Ranjet/env/lib/python3.9/site-packages/facebook_scraper/facebook_scraper.pyÚ__init__:   s    zFacebookScraper.__init__c                 C   s   || j jd< d S )Nr#   )r*   r'   )r,   Ú
user_agentr-   r-   r.   Úset_user_agentF   s    zFacebookScraper.set_user_agentc                 C   s*   |r| j j dd¡ n| j j dd¡ d S )NÚnoscriptÚ1Ú0)r*   ÚcookiesÚset)r,   r2   r-   r-   r.   Úset_noscriptI   s    zFacebookScraper.set_noscriptTc                 C   sB   | j  ||dœ|dœ¡ | jdddid ¡ }t d|› ¡ d S )N)ÚhttpÚhttps)ÚproxiesÚverifyzhttp://lumtest.com/myip.jsonÚAcceptzapplication/json)r'   zProxy details: )r+   r(   ÚgetÚjsonÚloggerÚdebug)r,   Úproxyr;   Úipr-   r-   r.   Ú	set_proxyO   s    ÿÿ
zFacebookScraper.set_proxy)ÚaccountÚreturnc                 K   s4   | |d< t tf|| jdœ|¤Ž}| jt|fi |¤ŽS )NÚscraper©rD   Ú
request_fn)r   r   r=   Ú_generic_get_postsr   ©r,   rD   ÚkwargsÚiter_pages_fnr-   r-   r.   Ú	get_postsX   s    zFacebookScraper.get_posts)Úpost_idrE   c                 K   sD   d|› }t  d|› ¡ |  |¡}t|j|| j|jd}| |¡S )NzGhttps://m.facebook.com/ufi/reaction/profile/browser/?ft_ent_identifier=ú	Fetching )Úfull_post_html)r?   r@   r=   r   ÚhtmlZextract_reactors)r,   rN   rK   Zreaction_urlÚresponseÚ	extractorr-   r-   r.   Úget_reactors]   s    ÿ
zFacebookScraper.get_reactorsc                 K   s,   t tf|| jdœ|¤Ž}| jt|fi |¤ŽS )NrG   )r   r   r=   rI   r   rJ   r-   r-   r.   Ú
get_photosf   s    zFacebookScraper.get_photos)ÚhashtagrE   c                 K   s<   | |d< t |d< ttf|| jdœ|¤Ž}| jt|fi |¤ŽS )NrF   Úbase_url)rV   rH   )r   r   r   r=   rI   r   )r,   rV   rK   rL   r-   r-   r.   Úget_posts_by_hashtagj   s    ÿÿÿz$FacebookScraper.get_posts_by_hashtag)rE   c                 c   sJ  | j j d¡dkrd|d< |D ]$}t|ƒ}| t¡rB| tt¡}| t¡rX| tt¡}| t¡snt	 
t|¡}||dœ}t d|› ¡ |  |¡}|j|d< d}d|v s°d	|v rÂ|jjd
dd}	n@|jjddd}	|	sæ|jjddd}	|jjdddrd}|j}	|	st d¡ n(|jjddd}
|
r|zt	 |	j d|
jd ¡¡}	W n0 tyz } zt |¡ W Y d }~n
d }~0 0 |rž| t|	| j||jd¡ nz| t	 
td¡¡rÎ| t|	| j||jd¡ nJd|v sâd	|v rþ| t|	| j||jd¡ n| t|	| j||jd¡ | d¡s,||d< |r>| dd ¡ |V  qd S )Nr2   r3   T©Zoriginal_request_urlÚpost_urlúRequesting page from: Zresponse_urlFz	/stories/z/story/z#story_viewer_content©Úfirstz[data-ft*="top_level_post_id"]zdiv.async_likezdiv.msgz:No raw posts (<article> elements) were found in this page.zdiv.ufiz	</footer>)rH   ÚoptionsrP   ú/groups/rZ   Úsource)r*   r5   r=   ÚstrÚ
startswithr   Úreplacer   r   r   r   r?   r@   ÚurlrQ   ÚfindÚwarningÚmake_html_elementÚ
ValueErrorr(   r   r   r   r   Úpop)r,   Z	post_urlsr^   Úremove_sourcerZ   rd   ÚpostrR   Z
photo_postÚelemZcomments_areaÚer-   r-   r.   Úget_posts_by_urlr   s’    






ÿ üÿüÿüÿ	üÿz FacebookScraper.get_posts_by_url)ÚwordrE   c                 K   s4   | |d< t tf|| jdœ|¤Ž}| jt|fi |¤ŽS )NrF   )ro   rH   )r   r   r=   rI   r   )r,   ro   rK   rL   r-   r-   r.   Úget_posts_by_searchÃ   s    z#FacebookScraper.get_posts_by_searchc                 k   s¸  |  d¡}d }t|ƒttfv r"|}| dd ¡}|sFt td|› d¡}|  d¡}d}|r´t 	d|› ¡ |   |¡}|j
 d¡}	t 	d	t|	ƒ› d
¡ |	D ]È}
|
jddd}|s²qš|
jddd}|rÌ|j}nd}|
jdddj  d¡}t d|¡}|r
t | ¡ d ¡}|
jddd}|r6t |jd ¡  d¡}nd}||j  d¡|j||dœ}|V  |d7 }qš|rx||krxd S t d|j¡}|r®t t| d¡¡}|r²||ƒ qTd S qTd S )NÚfriendsZ	start_urlú/z	/friends/Úrequest_url_callbackr   r[   z!div[class="timeline"] > div > divúFound z friendsz	h3>a,h1>aTr\   zspan.fcgÚ ú	i.profpicÚstyleúurl\('(.+)'\)za.touchable[data-store]ú
data-storeÚidÚhref)rz   ÚlinkÚnameÚprofile_pictureÚtagliner   zm_more_friends",href:"([^"]+)")r=   ÚtypeÚintÚfloatri   r   r   r   r?   r@   rQ   re   ÚlenÚtextÚattrsÚreÚsearchÚdecode_css_urlÚgroupsr>   ÚloadsÚgroup)r,   rD   rK   Z
friend_optÚlimitZ
friend_urlrs   Zfriends_foundrR   Úelemsrl   r}   r   r~   ÚmatchZuser_idZfriendÚmorer-   r-   r.   Úget_friendsÈ   s\    



û

zFacebookScraper.get_friendsc                 k   sî  |  d¡}d}|rêt d|› ¡ |   |¡}|j d¡rÖtdƒ}t |j|d … ¡}|d d D ]p}	|	d dkrž|	d	 ržtj	|	d	 t
d
}
|
 d¡}|
j}qb|	d dkrbt d|	d ¡}|rb| d¡}t |¡}qbn(|j d¡}t d|j¡}|rþ| d¡}t dt|ƒ› d¡ |D ]ž}|jdddj}|j  d¡}z|jdddj}W n   d }Y n0 |jdddj  d¡}t d|¡}|r˜t | ¡ d ¡}||||dœ}|V  |d7 }qt|ƒttfv rØ||krØd S |r|r||ƒ qd S )Nrs   r   r[   ú	for (;;);ÚpayloadÚactionsÚcmdÚappendrQ   ©rd   za.touchableÚscriptú/("\\/timeline\\/app_collection\\/more\\/[^"]+")Úcoder   z#timelineBody a.touchableú,href:"(/timeline/app_collection/more/[^"]+)"rt   z elemsÚstrongTr\   r{   zdiv.twoLinesrv   rw   rx   )r|   r}   r~   r   )r=   r?   r@   r„   rb   rƒ   r>   rŠ   r   rg   r   re   r†   r‡   r‹   rQ   r…   rˆ   r‰   r€   r   r‚   )r,   Úmore_urlrŒ   rK   rs   ÚcountrR   Úprefix_lengthÚdataÚactionÚelementr   rQ   rl   r}   r|   r   r~   rŽ   Úresultr-   r-   r.   Úget_collection   sh    

þ
ÿ
ÿ

üzFacebookScraper.get_collectionc           %   
   K   sH	  |  dd¡}i }| dd¡rXt d|› ¡ |  |¡}|jjddd}t||| jƒ ¡ }| d¡ ||d	< z(t	 
|j d
¡d j ¡ d ¡|d< W n< tyÐ } z$d |d< t d|› ¡ W Y d }~n
d }~0 0 z"t	 
|jjddddj¡|d< W n> ty2 } z$d |d< t d|› ¡ W Y d }~n
d }~0 0 zFd|› d}t d|› ¡ |  |¡}t	 
|jjdddj¡|d< W n> ty¸ } z$d |d< t d|› ¡ W Y d }~n
d }~0 0 |j d¡}	t|	ƒdkr|	d }
|  |
j d¡¡}t|j|| jƒ}| |jj¡|d< nDt|	ƒdkrð|	d }|j d¡|d < |d  d urª|  |j d¡¡}t|j|| jƒ}| |jj¡|d!< |	d }
|  |
j d¡¡}| |jj¡|d< nDd |d!< |	d }
|  |
j d¡¡}t|j|| jƒ}| |jj¡|d< nh|jjd"dd}|r4t d#|jd$ ¡}|r4t	 | ¡ d ¡|d!< |jjd%dd}|rX|jd& |d< t	 td|› d'¡}t d|› ¡ |  |¡}t d(|jj¡}|rª| d¡|d)< |jjdddj}d*|v rÔ| d*¡d }||d+< |jjd,dd}|s t d-¡ |S | d.¡D ]|}|jd/ddj}| d0¡r0d0}|d1v rðg }| d2¡D ]š}i }z|jd3ddjd |d4< W n   Y n0 |j d5¡}t|ƒdkr¦|\|d6< |d7< n0t|ƒd8krÌ|\|d6< |d7< |d9< n
|j|d6< | |¡ qH|||< q
|d:kr„g }| d;¡D ]n}i }z|jd3ddjd |d4< W n   Y n0 d5|jv r`|j d5¡\|d6< |d7< n
|j|d6< | |¡ q|||< q
|j d5¡dd … }t|ƒd8krÎ|d<krÎ|d |d |d d=œ||< n¸t|ƒdkrê|d ||< nœ|d>v rxt|ƒd dkrxi }tdt|ƒdƒD ]R}||d  d?krVd?|vrBg |d?< |d?  || ¡ n|| |||d  < q|||< nd5 |¡||< q
| d@¡r°t| j|fi |¤Žƒ|dA< | dB¡rèt| j d|› dCfdD| dB¡i|¤Žƒ|dE< | dF¡r t| j d|› dfdD| dF¡i|¤Žƒ|dG< | d)¡	rD| dH¡	rDt	 tdI|d) › dJ¡}t d|› ¡ |  |¡}i |dK< |j dL¡D ]>}|j d5¡\}}t	 
|¡}|dMkr®||dN< ||dK |< q~t	 tdO|d) › dP¡}t d|› ¡ |  |¡}g |dH< |j dQ¡D ].}|dH  |j|jd3ddj d¡dRœ¡ qt dS|j¡} | rR|  d¡} | 	rDt d| › ¡ |  | ¡}tdTƒ}!t! "|j|!d … ¡}"|"dU dV D ]¤}#|#dW dXk	r|#dY 	rt	j#|#dY tdZ}$|$ dQ¡D ].}|dH  |j|jd3ddj d¡dRœ¡ qÒn8|#dW d[kršt d\|#d] ¡} | rš|  d¡} t! "| ¡} qšqR|S )^Nzprofile.php?id=ru   Zallow_extra_requestsTr[   zG[data-ft*="top_level_post_id"]:not([data-sigil="m-see-translate-link"])r\   r`   Útop_postz a[data-store*='friends']>div>divéÿÿÿÿr   ZFriend_countz Friend_count extraction failed: ú)div[data-sigil*='profile-intro-card-log']zFollowed by)Ú
containingr]   ZFollower_countz"Follower_count extraction failed: rr   z?v=followingrO   zdiv[role='heading']ZFollowing_countz#Following_count extraction failed: za[href^='/photo.php']r   r{   r~   é   ÚtitleZcover_photo_textÚcover_photoz#div[data-sigil='cover-photo']>i.imgrx   rw   zimg.profpicÚsrcú/about/zentity_id:(\d+)rz   z | ÚNamezdiv#main_column,div.aboutmezNo about section foundzdiv[data-sigil='profile-card']ÚheaderZAbout)zWork, Educationzdiv.experienceÚar|   Ú
r„   r€   é   ÚyearzPlaces livedzdiv.touchableZRelationship)Útor€   Zsince)zContact Infoz
Basic InfoZ	EducationzFamily MemberszOther namesZWebsitesrq   ZFriendsÚ	followersz?v=followersrŒ   Z	FollowersZ	followingZ	FollowingÚlikesz$timeline/app_section/?section_token=z:2409997254Zlikes_by_categoryz(header[data-sigil="profile-card-header"]z	All LikesZlikes_countz*timeline/app_collection/?collection_token=z:2409997254:96z	div._1a5p©r}   r|   rš   r‘   r’   r“   r”   r•   rQ   r–   r—   r˜   r™   )$rc   r=   r?   r@   rQ   re   r   r   ri   r   Ú	parse_intr„   ÚsplitÚ	ExceptionÚerrorrƒ   r…   Zextract_photo_link_HQr†   r‡   rˆ   r‰   r   r   r‹   rf   rb   r•   ÚrangeÚjoinÚlistr   r£   r>   rŠ   rg   )%r,   rD   rK   r¢   rR   r¤   rm   Zfollowing_urlZfollowing_responseZphoto_linksÚprofile_photorS   rª   rŽ   ZprofpicÚ	about_urlr©   ÚaboutZcardr®   Z
experiencerl   ZxpÚbitsZplacesZplaceÚpairsÚiZ	likes_urlr   ÚcategoryZall_likes_urlrœ   rž   rŸ   r    r¡   r-   r-   r.   Úget_profile8  s¶   
þ
ÿ&ýÿ&
ÿ&ÿ
ÿ
ÿ






ÿÿ	÷


ÿÿÿÿ
ÿÿÿÿþ


þ
þÿ

þþÿ
ÿ
zFacebookScraper.get_profilec                 k   s  d|› d}|rt  d|› ¡ |  |¡}|j d¡rÚtdƒ}t |j|d … ¡}|d d D ]v}|d dkr–|d	 r–tj	|d	 t
d
}| d¡}	q`|d dkr`t d|d ¡}|r`| d¡}t |¡}| dd¡}q`n*|j d¡}	t d|j¡}|r| d¡}|	D ]}
|
jddd}|s&qt|j ¡ ƒ}|d  ¡ }d|jv }| d¡}t|ƒdkrzt t|d jd ¡}nd }|
jddd}|rš|j}nd }|
jddd}t |jd ¡d }|||
jdddjd  ||jt |¡||t t|
jd!ddjd ¡d"œ	V  qqd S )#Nrr   z/reviewsrO   r‘   r’   r“   r”   rc   rQ   r–   z!#page_suggestions_on_liking ~ divr—   z#see_more_cards_id","href":"([^"]+)"r™   r   ú\ru   z!see_more_cards_id",href:"([^"]+)"z!div[data-nt='FB:TEXT4']:has(span)Tr\   r   Ú
recommendsr¯   r¨   r{   z"div[data-nt='FB:FEED_TEXT'] span pzabbr[data-store*='time']ry   ÚtimeÚimgr«   za[href*='story'])	Úuser_urlÚusernamer~   r„   r®   rÈ   Ú	timestamprÇ   rZ   )r?   r@   r=   r„   rb   rƒ   r>   rŠ   r   rg   r   re   r†   r‡   r‹   rˆ   rc   rQ   r½   r¡   ÚitertextÚstripr   r   r…   r	   Úfromtimestamp)r,   ÚpagerK   rœ   rR   rž   rŸ   r    r¡   r   rl   Zheader_elemrÁ   rË   rÇ   ÚlinksrÊ   Z	text_elemr„   Zdate_elementrÈ   r-   r-   r.   Úget_page_reviews'  sn    
þÿ





ÿ÷z FacebookScraper.get_page_reviewsc              
   K   sä  i }d }zÐd|› d}t  d|› ¡ |  |¡}|jjdddj dd¡|d	< |jjd
dd}|jjdddj|d< |jjddd}|r´t d|j	d ¡}|r´t
 | ¡ d ¡|d< |jjddd}	|	rÖ|	j	d |d< W n0 ty }
 zt  |
¡ W Y d }
~
n
d }
~
0 0 zØd|› d}t  d|› ¡ |  |¡}|jjdddj dd¡|d	< |jjd
dd}d }z|jjdddj}W n¦   t  d¡ d|› d}t  d|› ¡ zf|  |¡}z|jjdddj}W n@   t  d¡ |jjdddj d¡}t
 |d ¡|d< Y n0 W n   Y n0 Y n0 |r~t |¡}| |d ¡ | d¡|d< | d g ¡D ] }|d! d"kr\|d# |d< q\z|jjd$ddj|d< W n: tyÒ }
 z t  |
¡ |  |¡}W Y d }
~
n
d }
~
0 0 |j d%¡D ]ê}|j d¡d }d&|v rt
 | d&d¡¡|d< | d'¡r2| d(¡d) |d'< |jd*dd}|rà|j	d+ }d,|v rb||d-< d.|v rtt|ƒjƒ d/¡d |d0< ||d1< | d2¡r¬| d2d¡|d3< | d4¡rà| d4d¡|d5< qà|j d6¡d7 j|d8< W n0 ty }
 zt  |
¡ W Y d }
~
n
d }
~
0 0 |rœt  |j	d9 ¡ t d:|j	d9 ¡}|rZt
 | ¡ d ¡|d;< |j	d9  d<¡}t|ƒd=krœt
 |d7 ¡|d>< t
 |d ¡|d?< | d@¡rà| j|fi |¤Ž|d@< | d@¡dAkràt
 |d@ ¡|d@< |S )BNrr   r¬   r[   r©   Tr\   z - Aboutru   r}   zmeta[name='description']z&#pages_msite_body_contents,div.aboutmerÀ   z)#msite-pages-header-contents i.coverPhotorx   rw   r   rª   z #msite-pages-header-contents imgr«   r¾   z - Homez"script[type='application/ld+json']zNo ld+json elementz
/communityz#page_suggestions_on_liking+divr°   r¨   r´   Úauthorz@typer€   ZinteractionStatisticZinteractionTypezhttp://schema.org/FollowActionZuserInteractionCountz/#pages_msite_body_contents>div>div:nth-child(2)r¦   z
 FollowerszPrice Rangeu    Â· r¥   úa[href]r{   Z
active_adsZactive_ads_linkzmaps.google.comÚuZmap_linkÚaddressztel:Zphonezmailto:Úemailzdiv[data-nt='FB:TEXT4']r   ZratingÚcontentu   \..+?(\d[\d,.]+).+Â·rµ   õ   Â·r±   Zpeople_talking_about_thisZcheckinsZreviewsÚ	generator)r?   r@   r=   rQ   re   r„   rc   r†   r‡   r…   r   rˆ   r‰   r¹   rº   r¸   Zconvert_numeric_abbrÚdemjsonÚdecoder(   ri   rÅ   rb   r   r   Úqueryr·   rƒ   rÒ   Zsafe_consume)r,   rÐ   rK   r¢   Údescr¿   Úresprª   rŽ   r¾   rm   rd   Zld_jsonZcommunity_respZlikes_and_followsÚmetaZinteractionrl   r„   r|   rÁ   r-   r-   r.   Úget_page_infoe  sÂ    
ÿ 


ÿ

ÿ
 

ÿ


 zFacebookScraper.get_page_infoc              
      s¾  |   d¡ d|› }t d|› ¡ |  |¡j}z|jdddjd }W n tyd   t 	d¡‚Y n0 t d|› ¡ |  |¡j}i ‰ t
 d	|¡ d
¡ˆ d< zJ|jdddjˆ d< |jdddjˆ d< |jddd}t |j¡ˆ d< W n ty   t 	d¡‚Y n0 |jdddj d¡}t d|› ¡ zb|  |¡j}|jddd}|r–|j d¡}t d|› ¡ z|  |¡j}W n   t 	d¡‚Y n0 n|}t
 d|jdddj d¡¡rÆ| d¡}n
| d¡}dd„ |D ƒˆ d< |jddd}|r„|jd }g }|rlt d|› ¡ |  |¡j}| d¡}| dd„ |D ƒ¡ t
 d |j¡}	|	rd|	 d
¡}nd }q‡ fd!d„|D ƒˆ d"< n
t d#¡ W n( tjy¸ }
 zW Y d }
~
n
d }
~
0 0 ˆ S )$Nr"   r_   r[   za[href*='?view=info']Tr\   r{   zUnable to resolve view=info URLz/groups/(\d+)r   rz   z	header h3r}   z
header divr€   z+div[data-testid='m_group_sections_members']Úmembersz+Unable to get one of name, type, or membersr¯   z(a[href*='listType=list_admin_moderator']zUnable to get admin listz/groups/members/searchz3div:nth-child(1)>div:nth-child(1) a:not(.touchable)z2div:nth-of-type(2)>div.touchable a:not(.touchable)z/div:first-child>div.touchable a:not(.touchable)c                 S   s*   g | ]"}|j tj|jd  dgddœ‘qS )r{   Zrefid)Z	blacklistr¶   )r„   r   Zfilter_query_paramsr…   ©Ú.0rm   r-   r-   r.   Ú
<listcomp>ø  s   ýþz2FacebookScraper.get_group_info.<locals>.<listcomp>Úadminsz+a[href*='listType=list_nonfriend_nonadmin']z%#root div.touchable a:not(.touchable)c                 S   s   g | ]}|j |jd  dœ‘qS )r{   r¶   )r„   r…   rã   r-   r-   r.   rå     ó    z"m_more_item",href:"([^"]+)c                    s   g | ]}|ˆ d  vr|‘qS )ræ   r-   )rä   Úm©r¢   r-   r.   rå     rç   Zother_memberszNo other members listed)r1   r?   r@   r=   rQ   re   r…   ÚAttributeErrorr    ÚUnexpectedResponser†   r‡   r‹   r„   r   r·   rŽ   Úextendrf   ÚLoginRequired)r,   r‹   rK   rd   rß   râ   Z
respAdminsræ   r   r   rm   r-   ré   r.   Úget_group_infoÊ  s‚    ÿ
ÿþþ
û


zFacebookScraper.get_group_infoc                 K   sì   |   d¡ |  d¡ |› d}t d|› ¡ |  |¡}|jjddd}|rt|d jd	 }t d|› ¡ |  |¡}|j d
¡}g }|D ]^}|jddd}	|	j}
|	jd	 }|jdddjd }| d¡d j}|
|||dœ}| 	|¡ qˆ|S )Nr"   Tz/shop/rO   rÔ   zSee More©r§   r¥   r{   zdiv.bezdiv.bl ar\   rÉ   r«   zdiv.bl)r}   r|   ÚimageÚprice)
r1   r7   r?   r@   r=   rQ   re   r…   r„   r•   )r,   rÐ   rK   rd   rß   Z
more_linksÚitemsÚresultsÚitemZ	link_elemr}   r|   rð   rñ   r¢   r-   r-   r.   Úget_shop  s.    ÿ




zFacebookScraper.get_shop)r‹   rE   c                 K   s6   |   d¡ ttf|| jdœ|¤Ž}| jt|fi |¤ŽS )Nr"   )r‹   rH   )r1   r   r   r=   rI   r   )r,   r‹   rK   rL   r-   r-   r.   Úget_group_posts.  s
    ÿzFacebookScraper.get_group_postsc                 C   sN   | j r
d S t d|j¡}|rJ| d¡d }|dkrDt d|› d¡ d| _ d S )Nz/"IntlCurrentLocale",\[\],{code:"(\w{2}_\w{2})"}r   r   r%   zFacebook language detected as z! - for best results, set to en_UST)Úhave_checked_localer†   r‡   r„   r‰   ÚwarningsÚwarn)r,   rR   rŽ   r$   r-   r-   r.   Úcheck_locale5  s    
ÿzFacebookScraper.check_localec              
   K   sÀ  z‚t |ƒ}| d¡s"t t|¡}| d¡rN| d¡ | jjf d|i|¤Ž}n| jjf d|i| j	¤|¤Ž}d}|ræt
 d¡D ]h}| d¡r|| dd¡|v r|t d|› d	|› ¡ t|ƒ}| ¡ |j_W d   ƒ q|1 sÚ0    Y  q||jj d
d¡ dd¡|j_| ¡  |  |¡ |jdkrnt d|¡ ¡ }|rnt td|› d¡}||dœ}t d|› ¡ |  |¡}d|jv rÀtt|jƒjƒ d¡d }	d|	› d|	› d}t d|› ¡ |  |¡}d|jv rÖ|  |¡}|j t¡r*|jjddds*d|jjvr*| jj d¡dkr*t  d|j› ¡ |jjdd d!rFt  d"¡ |jjd#dd}
d$d%g}g d&¢}d'|jv rŽ|jjd(d)d!rŽt! "d*¡‚|
r€|
j# $¡ |v r²t! %|
j#¡‚nÎ|
j# $¡ d+krÎt! &d,¡‚n²|
j# $¡ |v rìt! '|
j#¡‚n”d-|jj $¡ v r
t! "d*¡‚nvd.|jjv r$t! "d/¡‚n\|
j#d0ksv|j t td1¡¡sv|j t t(d1¡¡svd2|j#v r€|j d3¡s€t! )d4¡‚|W S  t*yº } zt +d5||¡ ‚ W Y d }~n
d }~0 0 d S )6Nr8   rk   rd   FÚ.z.htmlru   z
Replacing z content with z<!--z-->z1https://m.facebook.com/watch/?ref=watch_permalinkz\d+zstory.php?story_fbid=z!&id=1&m_entstream_source=timelinerY   r[   z/watch/Úvr   z&id=zS&m_entstream_source=video_home&player_suborigin=entry_point&player_format=permalinkrO   zcookie/consent-pager—   Tr\   r2   r3   z8Facebook served mbasic/noscript content unexpectedly on zh1,h2zUnsupported Browserrï   z#Facebook says 'Unsupported Browser'r©   zpage not foundzcontent not found)z(you can't use this feature at the momentz$you can't use this feature right nowu   youâ€™re temporarily blockedÚ
checkpointÚh1zWe suspended your accountzYour Account Has Been Disabledrº   z"Your request couldn't be processedz >your account has been disabled<zs>We saw unusual activity on your account. This may mean that someone has used your account without your knowledge.<zYour Account Has Been LockedzLog in to Facebook | FacebookÚloginz, log in to Facebook.z0article[data-ft],div.async_like[data-ft],div.msgz.A login (cookies) is required to see this pagez0Exception while requesting URL: %s
Exception: %r),ra   rb   r   r   r   r=   ri   r*   rk   r+   ÚosÚlistdirÚendswithrc   r?   r@   ÚopenÚreadrQ   Úraise_for_statusrú   rd   r†   r‡   r‹   r   r   rÝ   Úsubmit_formre   r5   rø   rù   r    ZAccountDisabledr„   ÚlowerZNotFoundrë   ZTemporarilyBannedr   rí   r
   Ú	exception)r,   rd   rK   rR   ÚDEBUGÚfilenameÚfrZ   rk   Zvideo_idr©   Znot_found_titlesZtemp_ban_titlesÚexr-   r-   r.   r=   A  s®    



,

þ




ÿþ
ýü
ÿ

ÿÿÿþýûÿúÿzFacebookScraper.getc                 C   sh   |j jdddj d¡}t | j|¡}|j  d¡}dd„ |D ƒ}| |¡ | jj	|fd|i| j
¤Ž}|S )	NÚformTr\   r    zinput[name][value]c                 S   s   i | ]}|j d  |j d “qS )r}   Úvalue)r…   )rä   rl   r-   r-   r.   Ú
<dictcomp>¤  rç   z/FacebookScraper.submit_form.<locals>.<dictcomp>rŸ   )rQ   re   r…   r=   r   r   rW   r(   r*   rk   r+   )r,   rR   Ú
extra_datar    rd   r   rŸ   r-   r-   r.   r     s    
zFacebookScraper.submit_form)r×   Úpasswordc           	      C   s°  |   | j¡}t d|jj¡}t|ƒdkr<| jj d|d ¡ |  	|||d dœ¡}|jj
ddd}|rpt |j¡‚d	|j ¡ v rtd
ƒ}|  	|d|i¡}|jj
ddd}|rÄ|j d¡rÄt |j¡‚|  	|ddi¡}d|j ¡ v r|  	|¡}|  	|dd i¡}|  	|ddi¡}d|j ¡ v s*d|jv rBtdƒ |  	|ddi¡}d|j ¡ v r\t d¡‚d| jjvr¬tddƒ}| |j¡ W d   ƒ n1 s˜0    Y  t d¡‚d S )Nzjs_datr","([^"]+)r   Zdatrr   )r×   ÚpassZ_fb_noscriptz#login_errorTr\   zenter login code to continuezEnter 2FA token: Zapprovals_coder›   z(The login code you entered doesn't matchZname_action_selectedZsave_devicezreview recent loginzsubmit[This wasn't me]zlogin approval neededrý   zLogin approval needed. From a browser logged into this account, approve this login from your notifications. Press enter once you've approved it.zsubmit[Continue]ÚContinuez*the password that you entered is incorrectz*The password that you entered is incorrectZc_userzlogin_error.htmlÚwzLogin unsuccessful)r=   rW   r†   ÚfindallrQ   rƒ   r*   r5   r6   r  re   r    Z
LoginErrorr„   r  Úinputrb   rd   r  Úwrite)	r,   r×   r  rR   Zcookies_valuesZlogin_errorÚtokenr›   r  r-   r-   r.   rÿ   ©  s@    ÿ
ÿ
,zFacebookScraper.loginc                 C   s,   z|   d¡ W dS  tjy&   Y dS 0 d S )Nzhttps://m.facebook.com/settingsTF)r=   r    rí   )r,   r-   r-   r.   Úis_logged_inÓ  s
    
zFacebookScraper.is_logged_iné   c                 k   sh  |d u ri }n&t |tƒr4tjddd dd„ |D ƒ}| jj d¡dkrNd|d< |rh|d	krhtjd
dd |d uræg }	d}
d}d}d}d}|ƒ D ]P}|D ]:}z |||| jd}|rÄ| dd ¡ |d d u rØ|
d7 }
|d |krèd}|d d u s|d |kr0|d7 }|| dkr&t 	d|¡ |V  W qš|d7 }||krdd}t 	d¡ t 	d|
¡ W  qØ|d d ur |d |	vr |	 
|d ¡ t d||d ¡ W qš tyÔ } zt d|¡ W Y d }~qšd }~0 0 qš|r qdqn~|d u rút d¡nt|ƒ}t d¡ t||ƒ ƒD ]J\}}t d|¡ |D ].}|||| jd}|rV| dd ¡ |V  q0qd S )Nz,The options argument should be a dictionary.r±   )Ú
stacklevelc                 S   s   i | ]
}|d “qS )Tr-   )rä   Úkr-   r-   r.   r  ê  rç   z6FacebookScraper._generic_get_posts.<locals>.<dictcomp>r2   r3   Tr¨   zHA low page limit (<=2) might return no results, try increasing the limitr   é2   F)r^   rH   r`   rÈ   r   zPosts scraped: %sz?Sequential posts behind latest_date reached. Stopping scraping.zPosts with null date: %sr„   zMSequential post #%s behind the date limit: %s. Ignored (in logs) from now on.zBAn exception has occured during scraping: %s. Omitting the post...zStarting to iterate pageszExtracting posts from page %s)Ú
isinstancer6   rø   rù   r*   r5   r=   ri   r?   Úinfor•   rf   r¹   r  Ú	itertoolsr   r»   r@   Úzip)r,   Zextract_post_fnrL   Z
page_limitr^   rj   Zlatest_dateZmax_past_limitrK   Zpinned_postsZnull_date_postsZtotal_scraped_postsZrecurrent_past_postsZ
show_everyÚdonerÐ   Zpost_elementrk   rm   ÚcounterrÃ   r-   r-   r.   rI   Ú  sŠ    
þ


ÿþýþ

z"FacebookScraper._generic_get_posts)NN)T)N))Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   rW   r)   r÷   r/   r1   r7   rC   ra   r   r   rM   r   ÚdictrT   rU   rX   rn   rp   r   r   r£   rÅ   rÒ   rá   rî   rõ   r   rö   rú   r=   r  rÿ   Úboolr  r   rI   r-   r-   r-   r.   r!   /   sJ   ý

		Q88 p>eK_	*ør!   )4r   ÚloggingÚurllib.parser   rø   r†   Ú	functoolsr   Útypingr   r   r>   Zdemjson3rÛ   r   r   r   r	   r   Úrequestsr
   Zrequests_htmlr   ru   r   Ú	constantsr   r   r   r   r   Z
extractorsr   r   r   r   r   r   Zfb_typesr   r   Zpage_iteratorsr   r   r   r   r   r    Ú	getLoggerr$  r?   r!   r-   r-   r-   r.   Ú<module>   s*    
