a
    xd2j                     @   s  d dl Z d dlZd dlmZmZmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d d	lmZ d dlZd d
lmZ d dlmZ d dlmZ d dl m!Z" d dl#m$Z% d dl#m&Z&m'Z' d dl(m)Z) dZ*dZ+dZ,g dZ-e Z.de._/de._0da1eed df Z2eee3 ed e3df Z4eed df Z5ee3e6f Z7e3Z8e3Z9e3Z:e3Z;e6Z<e3Z=eZ>e3Z?e'Z@ee3ee3 f ZAee3 ZBeZCedee3 f ZDee3 ZEz(e jFjGdksJ e jFjHdksJ W n eIy   eJdY n0 G dd deKZLG dd dZMG d d deMZNG d!d deMZOG d"d# d#ejPZQd0e9d$d%d&ZRd1d(d)ZSG d*d+ d+ejTZUG d,d- d-eUZVG d.d/ d/eUZWdS )2    N)urlparse
urlunparseurljoin)ThreadPoolExecutor)TimeoutError)partial)SetUnionListMutableMappingOptional)PyQuery)	UserAgent)Cleaner)etree)HtmlElement)tostring)
fromstring)search)findallResult)html_to_unicodezutf-8zhttps://example.org/zuMozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8)nextZmoreZolderTElementr   HTML      z#Requests-HTML requires Python 3.6+!c                   @   s   e Zd Zdd ZdS )
MaxRetriesc                 C   s
   || _ d S N)message)selfr    r!   E/var/www/html/Ranjet/env/lib/python3.9/site-packages/requests_html.py__init__A   s    zMaxRetries.__init__N)__name__
__module____qualname__r#   r!   r!   r!   r"   r   ?   s   r   c                	   @   s  e Zd ZdZdddeeeddddZee	ddd	Z
eedd
dZejeddddZe
jedddd	Z
eedddZejeddddZeedddZeedddZeedddZeedddZd2dddddeeeeeeddd Zdddd!eeeeed"d#d$Zeed%d&d'Z ee!d%d(d)Z"ee#dd*d+Z$d,d- Z%ee#dd.d/Z&eedd0d1Z'dS )3
BaseParseraJ  A basic HTML/Element Parser, for Humans.

    :param element: The element from which to base the parsing upon.
    :param default_encoding: Which encoding to default to.
    :param html: HTML from which to base the parsing upon (optional).
    :param url: The URL from which the HTML originated, used for ``absolute_links``.

    N)default_encodinghtml)r(   r)   urlreturnc                C   sH   || _ || _d| _|| _d | _t|tr2|tn|| _	d | _
d | _d S )NT)elementr*   skip_anchorsr(   	_encoding
isinstancestrencodeDEFAULT_ENCODING_html_lxml_pq)r    r,   r(   r)   r*   r!   r!   r"   r#   O   s    zBaseParser.__init__r+   c                 C   s,   | j r| j S tj| jdd | jS dS )zyBytes representation of the HTML content.
        (`learn more <http://www.diveintopython3.net/strings.html>`_).
        unicodeencodingN)r3   r   r   r,   stripr1   r9   r    r!   r!   r"   raw_htmlY   s    zBaseParser.raw_htmlc                 C   s0   | j r| jj| jddS tj| jdd S dS )zzUnicode representation of the HTML content
        (`learn more <http://www.diveintopython3.net/strings.html>`_).
        replaceerrorsr7   r8   N)r3   r<   decoder9   r   r   r,   r:   r;   r!   r!   r"   r)   c   s    zBaseParser.html)r)   r+   c                 C   s   | | j| _d S r   )r1   r9   r3   r    r)   r!   r!   r"   r)   m   s    c                 C   s
   || _ dS )zProperty setter for self.html.N)r3   rA   r!   r!   r"   r<   q   s    c                 C   sj   | j r| j S | jrXt| j| jd | _ z| jj| jdd W n tyV   | j| _ Y n0 | j rd| j S | jS )zzThe encoding string to be used, extracted from the HTML and
        :class:`HTMLResponse <HTMLResponse>` headers.
        r   r=   r>   )r.   r3   r   r(   r<   r@   r9   UnicodeDecodeErrorr;   r!   r!   r"   r9   v   s    zBaseParser.encoding)encr+   c                 C   s
   || _ dS )z"Property setter for self.encoding.N)r.   )r    rC   r!   r!   r"   r9      s    c                 C   s   | j du rt| j| _ | j S )z`PyQuery <https://pythonhosted.org/pyquery/>`_ representation
        of the :class:`Element <Element>` or :class:`HTML <HTML>`.
        N)r5   r   lxmlr;   r!   r!   r"   pq   s    
zBaseParser.pqc                 C   sH   | j du rBzt| jdd| _ W n" ty@   tj| j| _ Y n0 | j S )zs`lxml <http://lxml.de>`_ representation of the
        :class:`Element <Element>` or :class:`HTML <HTML>`.
        Nzhtml.parser)features)r4   
soup_parser)   
ValueErrorrD   r   r<   r;   r!   r!   r"   rD      s    
zBaseParser.lxmlc                 C   s
   | j  S )z\The text content of the
        :class:`Element <Element>` or :class:`HTML <HTML>`.
        )rE   textr;   r!   r!   r"   rI      s    zBaseParser.textc                 C   s
   | j  S )zsThe full text content (including links) of the
        :class:`Element <Element>` or :class:`HTML <HTML>`.
        )rD   Ztext_contentr;   r!   r!   r"   	full_text   s    zBaseParser.full_text*F)
containingcleanfirstr.   )selectorrL   rM   rN   r.   r+   c                   s   t |tr|g}|pjfdd|D }|rv| }g }|D ]$ t fdd|D rH|  qH|  |r| }g }|D ]  tt	
 j _|  qt||S )aC  Given a CSS Selector, returns a list of
        :class:`Element <Element>` objects or a single one.

        :param selector: CSS Selector to use.
        :param clean: Whether or not to sanitize the found HTML of ``<script>`` and ``<style>`` tags.
        :param containing: If specified, only return elements that contain the provided text.
        :param first: Whether or not to return just the first result.
        :param _encoding: The encoding format.

        Example CSS Selectors:

        - ``a``
        - ``a.someClass``
        - ``a#someID``
        - ``a[target=_blank]``

        See W3School's `CSS Selectors Reference
        <https://www.w3schools.com/cssref/css_selectors.asp>`_
        for more details.

        If ``first`` is ``True``, only returns the first
        :class:`Element <Element>` found.
        c                    s   g | ]}t |j d qS r,   r*   r(   )r   r*   ).0found)r9   r    r!   r"   
<listcomp>   s   z#BaseParser.find.<locals>.<listcomp>c                    s   g | ]}|   j  v qS r!   )lowerrJ   )rR   c)r,   r!   r"   rT          )r/   r0   r9   rE   copyanyappendreverselxml_html_tostringcleaner
clean_htmlrD   r<   _get_first_or_list)r    rO   rL   rM   rN   r.   elementselements_copyr!   )r,   r9   r    r"   find   s(    

zBaseParser.find)rM   rN   r.   )rO   rM   rN   r.   r+   c          	         s`   j |} fdd|D }|rV| }g }|D ] }tt|j |_|| q4t||S )a  Given an XPath selector, returns a list of
        :class:`Element <Element>` objects or a single one.

        :param selector: XPath Selector to use.
        :param clean: Whether or not to sanitize the found HTML of ``<script>`` and ``<style>`` tags.
        :param first: Whether or not to return just the first result.
        :param _encoding: The encoding format.

        If a sub-selector is specified (e.g. ``//a/@href``), a simple
        list of results is returned.

        See W3School's `XPath Examples
        <https://www.w3schools.com/xml/xpath_examples.asp>`_
        for more details.

        If ``first`` is ``True``, only returns the first
        :class:`Element <Element>` found.
        c                    s6   g | ].}t |tjs*t|j p$jd nt|qS rP   )r/   r   Z_ElementUnicodeResultr   r*   r9   r0   )rR   Z	selectionr.   r    r!   r"   rT     s   
z$BaseParser.xpath.<locals>.<listcomp>)	rD   xpathrX   r\   r]   r^   r<   rZ   r_   )	r    rO   rM   rN   r.   selectedr`   ra   r,   r!   rc   r"   rd      s    zBaseParser.xpath)templater+   c                 C   s   t || jS )zSearch the :class:`Element <Element>` for the given Parse template.

        :param template: The Parse template to use.
        )parse_searchr)   r    rf   r!   r!   r"   r     s    zBaseParser.searchc                 C   s   dd t || jD S )zSearch the :class:`Element <Element>` (multiple times) for the given parse
        template.

        :param template: The Parse template to use.
        c                 S   s   g | ]}|qS r!   r!   )rR   rr!   r!   r"   rT      rW   z)BaseParser.search_all.<locals>.<listcomp>)r   r)   rh   r!   r!   r"   
search_all  s    zBaseParser.search_allc                    s    fdd}t | S )u)   All found links on page, in as–is form.c               	   3   s^     dD ]N} z6| jd  }|rB|dr2 jsB|dsB|V  W q
 tyV   Y q
0 q
d S )Nahref#)zjavascript:zmailto:)rb   attrsr:   
startswithr-   KeyError)linkrl   r;   r!   r"   gen&  s    
zBaseParser.links.<locals>.gensetr    rr   r!   r;   r"   links"  s    
zBaseParser.linksc                 C   sV   t | }|d s t| j|S |d sRt | jj|d< dd | D }t|S |S )zMakes a given link absolute.netlocschemec                 s   s   | ]
}|V  qd S r   r!   rR   vr!   r!   r"   	<genexpr>A  rW   z,BaseParser._make_absolute.<locals>.<genexpr>)r   _asdictr   base_urlrx   valuesr   )r    rq   parsedr!   r!   r"   _make_absolute2  s    zBaseParser._make_absolutec                    s    fdd}t | S )zAll found links on page, in absolute form
        (`learn more <https://www.navegabem.com/absolute-or-relative-links.html>`_).
        c                  3   s    j D ]}  | V  qd S r   )rv   r   )rq   r;   r!   r"   rr   N  s    
z&BaseParser.absolute_links.<locals>.genrs   ru   r!   r;   r"   absolute_linksH  s    zBaseParser.absolute_linksc                 C   s|   | j ddd}|r,|jdd }|r,|S t| j }d|d ddd	 d |d< d
d |	 D }t
|}|S )z}The base URL for the page. Supports the ``<base>`` tag
        (`learn more <https://www.w3schools.com/tags/tag_base.asp>`_).baseT)rN   rl    /pathNc                 s   s   | ]
}|V  qd S r   r!   ry   r!   r!   r"   r{   g  rW   z&BaseParser.base_url.<locals>.<genexpr>)rb   rn   getr:   r   r*   r|   joinsplitr~   r   )r    r   resultr   r*   r!   r!   r"   r}   T  s    $zBaseParser.base_url)rK   )(r$   r%   r&   __doc___DefaultEncoding_HTML_URLr#   property_RawHTMLr<   	_BaseHTMLr)   setterr0   bytes	_Encodingr9   r   rE   r   rD   _TextrI   rJ   _Containingbool_Findrb   _XPathrd   r   r   _Resultrj   _Linksrv   r   r   r}   r!   r!   r!   r"   r'   E   sB   	
			&8 &r'   c                       sX   e Zd ZdZg dZddeedd fddZedd	d
Z	e
edddZ  ZS )r   zAn element of HTML.

    :param element: The element from which to base the parsing upon.
    :param url: The URL from which the HTML originated, used for ``absolute_links``.
    :param default_encoding: Which encoding to default to.
    )
r,   r*   r-   r(   r.   r3   r4   r5   _attrssessionN)r(   )r*   r(   r+   c                   s6   t t| j|||d || _|j| _|j| _d | _d S )NrQ   )superr   r#   r,   tagZ
sourcelinelinenor   )r    r,   r*   r(   	__class__r!   r"   r#   z  s
    zElement.__init__r6   c                    s.    fdd j D }dt jjd|S )Nc                    s"   g | ]}d  |t j| qS )z{}={})formatreprrn   )rR   attrr;   r!   r"   rT     rW   z$Element.__repr__.<locals>.<listcomp>z<Element {} {}> )rn   r   r   r,   r   r   )r    rn   r!   r;   r"   __repr__  s    zElement.__repr__c                 C   sR   | j du rLdd | j D | _ dD ]&}|| j v r$t| j |  | j |< q$| j S )zReturns a dictionary of the attributes of the :class:`Element <Element>`
        (`learn more <https://www.w3schools.com/tags/ref_attributes.asp>`_).
        Nc                 S   s   i | ]\}}||qS r!   r!   )rR   krz   r!   r!   r"   
<dictcomp>  rW   z!Element.attrs.<locals>.<dictcomp>)classrel)r   r,   itemstupler   )r    r   r!   r!   r"   rn     s    

zElement.attrs)r$   r%   r&   r   	__slots__r   r   r#   r0   r   r   _Attrsrn   __classcell__r!   r!   r   r"   r   m  s   c                	       s  e Zd ZdZdeedded eeee	dd fddZ
ed	d
dZdefe	eedddZdd Zdd Zdd Zdd Zdd Zddeeeeee eeef e	dddZd'eeeee	eeef e	d"d#d$Zd(eeeee	eeef e	d"d%d&Z  ZS ))r   a   An HTML document, ready for parsing.

    :param url: The URL from which the HTML originated, used for ``absolute_links``.
    :param html: HTML from which to base the parsing upon (optional).
    :param default_encoding: Which encoding to default to.
    NF)r   r*   r(   async_HTMLSessionAsyncHTMLSession)r   r*   r)   r(   r   r+   c                   sn   t |tr|t}tt| jt|dp<td| dd|||d |pZ|rVt pZt	 | _
d | _t| _d S )Nr)   z<html>z</html>)r,   r)   r*   r(   )r/   r0   r1   r2   r   r   r#   r   r   r   r   pageDEFAULT_NEXT_SYMBOLnext_symbol)r    r   r*   r)   r(   r   r   r!   r"   r#     s    


zHTML.__init__r6   c                 C   s   d| j dS )Nz
<HTML url=>)r*   r;   r!   r!   r"   r     s    zHTML.__repr__)fetchr   r+   c                    s@    fdd}| }|r$ |}ndS |r8j|S |S dS )zAttempts to find the next page, if there is one. If ``fetch``
        is ``True`` (default), returns :class:`HTML <HTML>` object of
        next page. If ``fetch`` is ``False``, simply returns the next URL.

        c                     s   j d d} | D ]z}|jdrd|jdg v rB|jd   S |jdg D ]}d|v rP|jd     S qPd|jd v r|jd   S qz| d jd W S  ty   Y d S 0 d S )	Nrk   )rL   rl   r   r   r   r   r   )rb   rn   r   
IndexError)
candidates	candidate_classr   r    r!   r"   get_next  s    zHTML.next.<locals>.get_nextN)r   r   r   )r    r   r   r   Z_HTML__nextr*   r!   r   r"   r     s    z	HTML.nextc                 c   s>   | }|V  z|j d| jdj}W q ty6   Y q:Y q0 qd S NTr   r   )r   r   r)   AttributeError)r    r   r!   r!   r"   __iter__  s    zHTML.__iter__c                 C   s   | j d| jdjS r   )r   r   r)   r;   r!   r!   r"   __next__  s    zHTML.__next__c                 C   s   | S r   r!   r;   r!   r!   r"   	__aiter__  s    zHTML.__aiter__c                    s2   | j d| jd}|sq.| j|I d H }|jS d S )NFr   )r   r   r   r   r)   )r    r*   responser!   r!   r"   	__anext__  s
    zHTML.__anext__c                 C   s   | j | d S r   )r   rZ   )r    r   r!   r!   r"   add_next_symbol  s    zHTML.add_next_symbol)script)r*   r   sleepwaitcontenttimeout	keep_pagec       	            sD  z| j  I dH }
t|I dH  |rJ|
j|dt|d idI dH  n(|
jd| j dt|d idI dH  d}|r|
|I dH }|rt|D ]&}|
j	
dI dH  t|I dH  qnt|I dH  |r|
j	dI dH  |
 I dH }|	s|
 I dH  d}
|||
fW S  ty>   |
 I dH  d}
Y dS 0 dS )zQ Handle page creation and js rendering. Internal use for render/arender methods. Nr   i  )optionszdata:text/html,ZPageDown)browserZnewPageasyncior   gotointr)   evaluaterangeZ	_keyboardZdownZupr   closer   )r    r*   r   
scrolldownr   r   reloadr   r   r   r   r   _r!   r!   r"   _async_render  s2    "(zHTML._async_render   皙?r   T       @)retriesr   r   r   r   r   r   c	                 C   s   | j j| _d}	| jtkrd}t|D ]V}
|	svz4| j j| j| j|||| j||||d	\}	}}W qz t	yr   Y qz0 q$ q|q$|	st
dt| j|	ttd}| j|j || _|S )a  Reloads the response in Chromium, and replaces HTML content
        with an updated version, with JavaScript executed.

        :param retries: The number of times to retry loading the page in Chromium.
        :param script: JavaScript to execute upon page load (optional).
        :param wait: The number of seconds to wait before loading the page, preventing timeouts (optional).
        :param scrolldown: Integer, if provided, of how many times to page down.
        :param sleep: Integer, if provided, of how many long to sleep after initial render.
        :param reload: If ``False``, content will not be loaded from the browser, but will be provided from memory.
        :param keep_page: If ``True`` will allow you to interact with the browser page through ``r.html.page``.

        If ``scrolldown`` is specified, the page will scrolldown the specified
        number of times, after sleeping the specified amount of time
        (e.g. ``scrolldown=10, sleep=1``).

        If just ``sleep`` is provided, the rendering will wait *n* seconds, before
        returning.

        If ``script`` is specified, it will execute the provided JavaScript at
        runtime. Example:

        .. code-block:: python

            script = """
                () => {
                    return {
                        width: document.documentElement.clientWidth,
                        height: document.documentElement.clientHeight,
                        deviceScaleFactor: window.devicePixelRatio,
                    }
                }
            """

        Returns the return value of the executed  ``script``, if any is provided:

        .. code-block:: python

            >>> r.html.render(script=script)
            {'width': 800, 'height': 600, 'deviceScaleFactor': 1}

        Warning: the first time you run this method, it will download
        Chromium into your home directory (``~/.pyppeteer``).
        NF	r*   r   r   r   r   r   r   r   r   1Unable to render the page. Try increasing timeoutr*   r)   r(   )r   r   r*   DEFAULT_URLr   looprun_until_completer   r)   	TypeErrorr   r   r1   r2   __dict__updater   )r    r   r   r   r   r   r   r   r   r   ir   r   r)   r!   r!   r"   render  s"    -

4zHTML.renderc	                    s   | j jI dH | _d}	| jtkr"d}t|D ]R}
|	sxz0| j| j|||| j||||d	I dH \}	}}W q| tyt   Y q|0 q* q~q*|	stdt	| j|	
ttd}| j|j || _|S )z1 Async version of render. Takes same parameters. NFr   r   r   )r   r   r*   r   r   r   r)   r   r   r   r1   r2   r   r   r   )r    r   r   r   r   r   r   r   r   r   r   r   r   r)   r!   r!   r"   arenderd  s"    
0zHTML.arender)r   Nr   Fr   Tr   F)r   Nr   Fr   Tr   F)r$   r%   r&   r   r   r2   r	   r0   r   r   r#   r   r   _NextSymbol_Nextr   r   r   r   r   r   r   floatr   r   r   r   r   r!   r!   r   r"   r     s   ,),'$Gc                       sT   e Zd ZdZed dd fddZeeddd	Ze	ed d
ddZ
  ZS )HTMLResponsezAn HTML-enabled :class:`requests.Response <requests.Response>` object.
    Effectively the same, but with an intelligent ``.html`` property added.
    r   N)r   r+   c                    s   t t|   d | _|| _d S r   )r   r   r#   r3   r   )r    r   r   r!   r"   r#     s    zHTMLResponse.__init__r6   c                 C   s&   | j s t| j| j| j| jd| _ | j S )N)r   r*   r)   r(   )r3   r   r   r*   r   r9   r;   r!   r!   r"   r)     s    zHTMLResponse.htmlr   c                 C   s   | |d}|j |j  |S )Nr   )r   r   )clsr   r   Zhtml_rr!   r!   r"   _from_response  s    
zHTMLResponse._from_response)r$   r%   r&   r   r	   r#   r   r   r)   classmethodr   r   r!   r!   r   r"   r     s   r   r6   c                 C   s   t s| rt a | rt |  S tS )zReturns an apparently legit user-agent, if not requested one of a specific
    style. Defaults to a Chrome-style User-Agent.
    )	useragentr   DEFAULT_USER_AGENT)styler!   r!   r"   
user_agent  s    r   Fc                 C   s.   |r&z
| d W S  t y"   Y d S 0 n| S d S )Nr   )r   )lrN   r!   r!   r"   r_     s    

r_   c                       sN   e Zd ZdZdddgfeeed fddZeddd	Ze	d
d Z
  ZS )BaseSessionzd A consumable session, for cookie persistence and connection pooling,
    amongst other things.
    Tz--no-sandbox)mock_browserverifybrowser_argsc                    s<   t    |rt | jd< | jd | j || _|| _d S )Nz
User-Agentr   )	r   r#   r   headershooksrZ   response_hookr   _BaseSession__browser_args)r    r   r   r   r   r!   r"   r#     s    
zBaseSession.__init__r6   c                 K   s   |j st|_ t|| S )z= Change response enconding and replace it by a HTMLResponse. )r9   r2   r   r   )r    r   kwargsr!   r!   r"   r     s    zBaseSession.response_hookc                    s.   t | ds(tj| j d| jdI d H | _| jS )N_browserT)ZignoreHTTPSErrorsZheadlessargs)hasattr	pyppeteerZlaunchr   r   r  r;   r!   r!   r"   r     s    
zBaseSession.browser)r$   r%   r&   r   r   listr#   r   r   r   r   r   r!   r!   r   r"   r     s   r   c                       s8   e Zd Z fddZe fddZ fddZ  ZS )r   c                    s   t t| jf i | d S r   )r   r   r#   )r    r   r   r!   r"   r#     s    zHTMLSession.__init__c                    s>   t | ds8t | _| j r&td| jt j| _	| j	S )Nr  zSCannot use HTMLSession within an existing event loop. Use AsyncHTMLSession instead.)
r  r   get_event_loopr   Z
is_runningRuntimeErrorr   r   r   r  r;   r   r!   r"   r     s    


zHTMLSession.browserc                    s*   t | dr| j| j  t   dS z* If a browser was created close it first. r  N)r  r   r   r  r   r   r;   r   r!   r"   r     s    
zHTMLSession.close)r$   r%   r&   r#   r   r   r   r   r!   r!   r   r"   r     s   r   c                       sH   e Zd ZdZded fddZ fddZ fd	d
Zdd Z  Z	S )r   z An async consumable session. NT)r   c                    s0   t  j|i | |pt | _t|d| _dS )a(   Set or create an event loop and a thread pool.

            :param loop: Asyncio loop to use.
            :param workers: Amount of threads to use for executing async calls.
                If not pass it will default to the number of processors on the
                machine, multiplied by 5. )max_workersN)r   r#   r   r  r   r   thread_pool)r    r   workersr   r  r   r   r!   r"   r#     s    zAsyncHTMLSession.__init__c                    s*   t t jg|R i |}| j| j|S )z7 Partial original request func and run it in a thread. )r   r   requestr   Zrun_in_executorr
  )r    r  r   funcr   r!   r"   r    s    zAsyncHTMLSession.requestc                    s(   t | dr| j I dH  t   dS r  )r  r  r   r   r;   r   r!   r"   r     s    
zAsyncHTMLSession.closec                 G   s2   dd |D }| j t|\}}dd |D S )z Pass in all the coroutines you want to run, it will wrap each one
            in a task, run it and wait for the result. Return a list with all
            results, this is returned in the same order coros are passed in. c                 S   s   g | ]}t | qS r!   )r   Zensure_future)rR   coror!   r!   r"   rT     s   z(AsyncHTMLSession.run.<locals>.<listcomp>c                 S   s   g | ]}|  qS r!   )r   )rR   tr!   r!   r"   rT     rW   )r   r   r   r   )r    ZcorosZtasksdoner   r!   r!   r"   run  s
    zAsyncHTMLSession.run)NNT)
r$   r%   r&   r   r   r#   r  r   r  r   r!   r!   r   r"   r     s     r   )N)F)Xsysr   urllib.parser   r   r   concurrent.futuresr   Zconcurrent.futures._baser   	functoolsr   typingr   r	   r
   r   r   r  requestsZpyqueryr   Zfake_useragentr   Zlxml.html.cleanr   rD   r   Z	lxml.htmlr   r   r\   Zlxml.html.soupparserr   rG   parser   rg   r   r   Zw3lib.encodingr   r2   r   r   r   r]   Z
javascriptr   r   r   r0   r   r   r   r   r   Z
_UserAgentr   r   r   r   Z_LXMLr   Z_Searchr   r   r   r   r   version_infomajorminorAssertionErrorr  	Exceptionr   r'   r   r   Responser   r   r_   Sessionr   r   r   r!   r!   r!   r"   <module>   s~     *( m

!