o
    sg                     @   s
  d Z ddlZddlZejd ddlZddlmZ ddlm	Z	 e	  ddl
mZmZ dddd	d
ddZG dd dZedkrdZeeZeej dZeeZeej dZeeZeej dZeeZeej dZeeZeej dZeeZeej dS dS )zModule to parse a url    N.)urlparse)load_dotenv)MAX_URL_DEPTHlanguages_to_checkzlMozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.361z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zen-US,en;q=0.5zgzip, deflate)z
User-AgentzUpgrade-Insecure-RequestsDNTAcceptzAccept-LanguagezAccept-Encodingc                   @   s8   e Zd ZdZddefddZdd Zdd	 Zd
d ZdS )
parsed_urlzClass to parse a urlTurlc                 C   s  |  | _d| _| | |r|  }|r|| _| | j | jjdks*| jjdkr.g | _n,| jj	ddd  | _t
| jtkrNd| jtd  | jt< | jd tv rWdnd| _| jj	dd 	d}t
|dkrs| jj| _d S t
|d	krd|| _d S t
|d	krd
|v r|d
}d||d d  | _d S d|v r|d}d||d d  | _d S d|dd  | _d S d S )NF/    r   Twww.r      cocom)lowersupplied_urllanguage_urlcheck_prefixcheck_if_redirectr   r
   path
componentssplitlenr   joinr   hostnamedomainindex)selfr   check_redirectredirect_url
url_splitsco_index r%   ,/var/www/html/XCapMarket/utils/url_parser.py__init__   s>   




zparsed_url.__init__c                 C   s   d|vrd|vrd| }t dd|}d}t|dk r!t|d }d|d| vr8|d	}|d d
 |d  }t|| _| jjd	 | jj | _dS )zCheck if the url has a prefixzhttps://zhttp://z	www[0-9].r         r   r   ://://www.N)	resubr   r   r   r
   schemer   r   )r    r   www_char_check	url_splitr%   r%   r&   r   M   s   

zparsed_url.check_prefixc                 C   s   d| j v r| jS dS )zt
        Args:
            link (str): The link to check

        Returns:
            link (str): The link
        zlinkedin.comF)r   r   )r    r%   r%   r&   check_link_linkedin_   s   
	zparsed_url.check_link_linkedinc              	   C   s   | j | j| jddg}|D ]D}ztj|ddtd}|jW   S  tjjy*   Y q tjjy4   Y q tjj	y>   Y q tjj
yH   Y q tjjyR   Y qw dS )zCheck if the url redirectsr+   r*   Tr   )allow_redirectstimeoutheadersF)r   r   replacerequestsheadHEADERS
exceptionsReadTimeoutConnectTimeoutConnectionError
InvalidURLMissingSchema)r    urls_to_checkurl_to_checkrr%   r%   r&   r   m   s,   zparsed_url.check_if_redirectN)T)	__name__
__module____qualname____doc__strr'   r   r1   r   r%   r%   r%   r&   r
      s    /r
   __main__zhttps://www3.proproductsllc.comzhttp://ir.mdjmjh.com/zhttp://www.intel.com/zhttps://about.gitlab.com/zhttps://www.car2go.comzhttps://www.altlaw.co.uk/)rE   sysr6   r   appendr,   urllib.parser   dotenvr   configs.configr   r   r8   r
   rB   url1
url_parsedprintr   url2url3url4url5url6r%   r%   r%   r&   <module>   sJ    
m




