j^>UddlZddlmZddlmZmZmZmZmZer ddl m Z m Z m Z hdZ eeed<GddeZGd d Zy) N)platform) TYPE_CHECKINGAnyOptional TypedDictUnion)Browser CDPSessionPage> ::markerbrsvgbodyheadhtmlmetapathstyletitleiframescriptblack_listed_elementsceZdZUdZeed<eed<eeed<eeed<eeed<e ed<eed<eed <eed <eed <y ) ElementInViewPortzIA typed dictionary containing information about elements in the viewport. node_indexbackend_node_id node_name node_value node_meta is_clickableorigin_xorigin_ycenter_xcenter_yN) __name__ __module__ __qualname____doc__str__annotations__intrlistbooli/var/www/html/Fitness-lenito-AI-main/venv/lib/python3.12/site-packages/langchain/chains/natbot/crawler.pyrrsHSO} CyMMMMr/rceZdZdZddZdeddfdZdeddfdZd eee fddfd Z d eee fd eddfd Z dd Z de efdZy)Crawlera A crawler for web pages. **Security Note**: This is an implementation of a crawler that uses a browser via Playwright. This crawler can be used to load arbitrary webpages INCLUDING content from the local file system. Control access to who can submit crawling requests and what network access the crawler has. Make sure to scope permissions to the minimal permissions necessary for the application. See https://python.langchain.com/docs/security for more information. returnNc: ddlm}|jjj d|_|j j|_|jjddd||y#t$r}d}t||d}~wwxYw) Nr)sync_playwrightz\Could not import playwright python package. Please install it with `pip install playwright`.F)headlessii8)widthheight) playwright.sync_apir5 ImportErrorstartchromiumlaunchbrowsernew_pagepageset_viewport_size)selfr5emsgs r0__init__zCrawler.__init__?s * ;   # # % . . 5 5u 5 E ,,//1  ##dd$CD   *C c" )  *sA>> BBBurlc|jjd|vr|nd|z|jjj|j|_i|_y)Nz://zhttp://)rF)r@gotocontextnew_cdp_sessionclientpage_element_buffer)rBrFs r0 go_to_pagezCrawler.go_to_pagePsH %3,3IODii''77 B #% r/ directionc|dk(r|jjdy|dk(r|jjdyy)Nupz(document.scrollingElement || document.body).scrollTop = (document.scrollingElement || document.body).scrollTop - window.innerHeight;downz(document.scrollingElement || document.body).scrollTop = (document.scrollingElement || document.body).scrollTop + window.innerHeight;)r@evaluate)rBrNs r0scrollzCrawler.scrollUsI   II  X & II  X !r/id_cd}|jj||jjt |}|r1|d}|d}|jj j ||ytdy)Nz links = document.getElementsByTagName("a"); for (var i = 0; i < links.length; i++) { links[i].removeAttribute("target"); } r#r$zCould not find element)r@rRrLgetr+mouseclickprint)rBrTjselementxys r0rXz Crawler.click_so 2**..s3x8 z*Az*A IIOO ! !!Q ' * +r/textcp|j||jjj|y)N)rXr@keyboardtype)rBrTr^s r0raz Crawler.typers% 3 %r/cN|jjjdy)NEnter)r@r`press)rBs r0enterz Crawler.entervs   )r/c KLMN|j}|j}tj}g}|jd}tdk(r|dk(rd}|jd}|jd}|jd}|jd} ||z} || z} d} d} |j d d d | d d | d d d|j jdgddd}|dN|dd }|d}|d}|d}|d}|dM|dLt|dd}|d}|d}|d}|d}|d}|d }d }i}g}d!d"i}d!d"i}d#ttd$ttd%tfd&} dtttfd'ttd%tttffNfd( }!d)ttt tttffd*td+td#ttd,td%t tttff KLMNfd- Kt#LD]2\}"}#M|"}$N|#j%}%K|d.|"|%|$\}&}'K|d/|"|%|$\}(}) |j'|"}|%t*vrX||\}*}+},}-|*|z}*|+|z}+|,|z},|-|z}-|*}.|+}/|*|,z}0|+|-z}1|.| kxr|0|k\xr |/| kxr|1|k\}2|2sg}3|!||"gd0}4|&xs|(}5|5sdn|&r t|'n t|)}6|5sdn|j-t|6g}7|%d1k(r'|5r%|7r#N||"}8|8d2vr |7j d3|8d4nj|%d5k(r|4j/d3d6k(s|%d/k(rd/}%|4j1d3d|4D]3}9|5r|7r|7j d7|9|4|9d8 |3j |4|95d}:||"d k\rN||"}:|:d9k(r2|%d5k(r+|"|vr'|:%|j'|"};||;}<|;d k\r ||D]o}?|?j/d<}@|?j/d#}%|?j/d=}:|?j/d>}A|?j/d?}B|:r|:d@ndA}CdA}D@|vrd|@D]\}E|Ej/d3}F|Ej/d}G|Fd7k(r+Br)Ej/dB}HBj |HdCGdDUCGd@z }C^Brd@j3B}Id@|I}DCdAk7rCj5}C| |%A}J|Jd/k7sDdAk(rJdEvrCj5dAk(r&|?||><CdAk7r!|=j dFJdG|>DdHCdI|JdH n|=j dFJdG|>DdJ|>dz }>rt7dKtj|z d dL|=S#t($rYwxYw)MNzwindow.devicePixelRatiodarwinzwindow.pageYOffsetzwindow.pageXOffsetzwindow.screen.widthzwindow.screen.heightrz [scrollbar z0.2f-z%])r\r]r^zDOMSnapshot.captureSnapshotT)computedStylesincludeDOMRectsincludePaintOrderstrings documentsnodes backendNodeId attributes nodeValue parentIndexnodeName isClickableindex inputValuevaluelayout nodeIndexboundsz-1FNrhas_click_handlerr3c8|dk(ry|dk(ry|dk(ry|dk(s|ryy)Nalinkinputimgbuttonr^r.)rr~s r0 convert_namez#Crawler.crawl..convert_names6CG#E!X%):r/keysci}tt|fdzD]6\}}|dkr |}|}||vs|||<|j||r4|cS|S)Nrir)zipiterremove)rrrvalues key_index value_indexkeyryrns r0find_attributesz&Crawler.crawl..find_attributessuF,/$z2B1Dq1H,I(K?i( ,$;"'F3KKK$% -JMr/ hash_treetagnode_id parent_idct|}||vr' |j} |} |||||||\}} ||k(rd|f} n |rd| f} nd} | |t|<| S)NTr})r)lower)rrrrr parent_id_str parent_namegrand_parent_idis_parent_desc_anchor anchor_idryadd_to_hash_tree node_namesparentrns r0rz'Crawler.crawl..add_to_hash_trees NMI-%j&;<BBD "("3 sI{O09/G , !9C597O%y)',Ic'l #Lr/rr)ra placeholderz aria-labelraltz#text>•|ra)raryrsubmit attribute)rarryr>rr) rrrrrr r!r"r#r$rrr r rz="">rrrtextareazzParsing time: z seconds)r@rLtimerRrappendrKsendsetrr)r-dictr+rr,tuple enumeraterrw ValueErrorr setdefaultrVpopjoinstriprY)OrBr@rLr;page_state_as_textdevice_pixel_ratiowin_upper_boundwin_left_bound win_width win_heightwin_right_boundwin_lower_boundpercentage_progress_startpercentage_progress_endtreedocumentrprrrrr input_valueinput_value_indexinput_value_valuesrzlayout_node_indexr|cursor child_nodeselements_in_view_portanchor_ancestrybutton_ancestryrrrwnode_name_index node_parentris_ancestor_of_anchorris_ancestor_of_button button_idr\r]r7r8elem_left_boundelem_top_boundelem_right_boundelem_lower_boundpartially_is_in_viewport meta_dataelement_attributesancestor_exceptionancestor_node_key ancestor_noder^relement_node_valuenode_input_text_index text_indexelements_of_interest id_counterr[rnode_is_clickablenode_meta_data inner_textrchild entry_type entry_value entry_key meta_stringconverted_node_namerrrrnsO @@@@r0crawlz Crawler.crawlysyy"66 $(MM2K$L x $6!$;!" !%/C!D $ .B C==)>?  MM*@A !/)!;!0:!= %&!"#!!%&?%EQG^_cFddfg  {{ )!dQU V #'y/#' #4Q#7 ( 1*/*@05l0C %*;%7 !&}!5%*:%6 !$U=%9'%B!C &+L&9 '27';(3G(<!)(!3'-k':)/)979 9;BF AVBF AV } '~    S#X .23i  #s(^ ($ CtXc]':!;;<$ $ $  } $   $  4#& ' $ $ L'0 &; "E? -K'.'?'E'E'GI/?eY 0 , !90@5)[0 , !9 *007 11$*6N !Q5& # #A # #A ' 'E ( (FON 5y  6z  /18$68"_48%7 %,#%I"15!#X" "7!O:O *)^^ * ++C0A,BBG  G#(:}z%01<'$$ft%DE(-?-C-CF-Kx-W(* (I&**.C)m%,,(3'*);C)@"(();C)@A."& % A%%,Z->%?"&#-W$..&.(9(?(?(F%/0EF (A-*/)0)<&"y'G ! ( ("%e*'6u'=!*"4!*$)\$9 #A #A #AO 4 #A!$4 5  g'rsK == #s3x   N$N$r/