dtjDddlZddlZddlZddlZddlZddlZddlmZddlmZddl m Z ddl Z ddl m Z mZddlmZmZejdZGd d eZGd d e ZdS) N)suppress)datetime)quote)AbstractBufferedFileAbstractFileSystem)infer_storage_optionstokenizewebhdfscheZdZdZeejZdZ d%fd Z e dZ d Z d&d Z d'd ZedZedZedZdZdZdZd(dZdZdZdZd)dZdZdZdZd*dZdZ dZ!d(dZ"d Z#d(d!Z$d"Z%d#Z&d$Z'xZ(S)+WebHDFSa Interface to HDFS over HTTP using the WebHDFS API. Supports also HttpFS gateways. Four auth mechanisms are supported: insecure: no auth is done, and the user is assumed to be whoever they say they are (parameter ``user``), or a predefined value such as "dr.who" if not given spnego: when kerberos authentication is enabled, auth is negotiated by requests_kerberos https://github.com/requests/requests-kerberos . This establishes a session based on existing kinit login and/or specified principal/password; parameters are passed with ``kerb_kwargs`` token: uses an existing Hadoop delegation token from another secured service. Indeed, this client can also generate such tokens when not insecure. Note that tokens expire, but can be renewed (by a previously specified user) and may allow for proxying. basic-auth: used when both parameter ``user`` and parameter ``password`` are provided. )r webHDFSFNTc  |jrdStjdi| | rdndd|d|d|_||_|pi|_i|_| pi|_|||td||jd<||_ ||_ ||td n | ||jd <| ||jd <|r|td | |_ | |_ | d t|||_dS)a Parameters ---------- host: str Name-node address port: int Port for webHDFS kerberos: bool Whether to authenticate with kerberos for this connection token: str or None If given, use this token on every call to authenticate. A user and user-proxy may be encoded in the token and should not be also given user: str or None If given, assert the user name to connect with password: str or None If given, assert the password to use for basic auth. If password is provided, user must be provided also proxy_to: str or None If given, the user has the authority to proxy, and this value is the user in who's name actions are taken kerb_kwargs: dict Any extra arguments for HTTPKerberosAuth, see ``_ data_proxy: dict, callable or None If given, map data-node addresses. This can be necessary if the HDFS cluster is behind a proxy, running on Docker or otherwise has a mismatch between the host-names given by the name-node and the address by which to refer to them from the client. If a dict, maps host names ``host->data_proxy[host]``; if a callable, full URLs are passed, and function must conform to ``url->data_proxy(url)``. use_https: bool Whether to connect to the Name-node using HTTPS instead of HTTP session_cert: str or Tuple[str, str] or None Path to a certificate file, or tuple of (cert, key) files to use for the requests.Session session_verify: str, bool or None Path to a certificate file to use for verifying the requests.Session. kwargs Nhttpshttpz://:z /webhdfs/v1z_If passing a delegation token, must not set user or proxy_to, as these are encoded in the token delegationzQIf passing a password, the user must also beset in order to set up the basic-authz user.namedoaszJIf using Kerberos auth, do not specify the user, this is handled by kinit.webhdfs_)_cachedsuper__init__urlkerb kerb_kwargsparsproxy ValueErroruserpassword session_certsession_verify_connectr _fsid)selfhostportkerberostokenr!r"proxy_tor data_proxy use_httpsr#r$kwargs __class__s \/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/fsspec/implementations/webhdfs.pyrzWebHDFS.__init__.ssr <  F""6"""!*6ggSS4SS$SSS &," %2  8#7  ',DIl #    | < )- +&   (DIf   (2  ), 6t 4 466 c|jSN)r&r's r1fsidz WebHDFS.fsids zr2c@tj|_|jr|j|j_|j|j_|jrddlm }|di|j |j_ |j /|j *ddlm}||j |j |j_ dSdSdS)Nr)HTTPKerberosAuth) HTTPBasicAuthr)requestsSessionsessionr#certr$verifyrrequests_kerberosr8rauthr!r" requests.authr9)r'r8r9s r1r%zWebHDFS._connects'))   2 $ 1DL "1  9 E : : : : : : 0 0 D D43C D DDL  9 T]%> 3 3 3 3 3 3 - di G GDL    ! %>%>r2getc |||nd}||jt|dz}|}||j||d<t d|||j |||||} | j dvr | } | dd } | dd } | d vrt| | d vrt| | d vrt!| t#| #tt$f$rYnwxYw| | S)Nz/=)safeopzsending %s with %s)methodrparamsdataallow_redirects)iiiiiRemoteExceptionmessage exception)IllegalArgumentExceptionUnsupportedOperationException)SecurityExceptionAccessControlException)FileNotFoundException)_strip_protocol _apply_proxyrrcopyupdaterupperloggerdebugr<request status_codejsonr PermissionErrorFileNotFoundError RuntimeErrorKeyErrorraise_for_status) r'rFrGpathrIredirectr/rargsouterrmsgexps r1_callz WebHDFS._calls-1-=t##D)))25D+A+A+A ABB{{}} DIXXZZT  )3777l""<<>>$ #   ?7 7 7 ,hhjj+,Y7+,[9WWW$S//)KKK)#...555+C000&s+++)      s"0EE.-E.rbc L|p|j}t|||||j|||S)a^ Parameters ---------- path: str File location mode: str 'rb', 'wb', etc. block_size: int Client buffer size for read-ahead or write buffer autocommit: bool If False, writes to temporary file that only gets put in final location upon commit replication: int Number of copies of file on the cluster, write mode only permissions: str or int posix permissions, write mode only kwargs Returns ------- WebHDFile instance )mode block_sizetempdir autocommit replication permissions) blocksize WebHDFilern)r'rbrlrmrorprqr/s r1_openz WebHDFS._opensBB 14>   !L!##    r2cV|d|d<|d|d<|S)Ntypelengthsize)lower)infos r1 _process_infozWebHDFS._process_infos-F|))++V H~V  r2c,t|dS)Nrb)r )clsrbs r1rSzWebHDFS._strip_protocols$T**622r2ct|}|dd|ddd|vr|d|d<|S)Nrbprotocolusernamer!)r pop)urlpathres r1_get_kwargs_from_urlszWebHDFS._get_kwargs_from_urlss\#G,,   D!!!   ''*--CK r2c|d|}|d}||d<||S)N GETFILESTATUSrb FileStatusname)rir\r{)r'rbrerzs r1rzz WebHDFS.info sFjjtj44xxzz,'V !!$'''r2c||}|dd}|tj|dz St d)z=Return the created timestamp of a file as a datetime.datetimemodificationTimeNz5Could not retrieve creation time (modification time).rzrBr fromtimestampr_r'rbrzmtimes r1createdzWebHDFS.createdsQyy+T22  )%$,77 7RSSSr2c||}|dd}|tj|dz St d)z>Return the modified timestamp of a file as a datetime.datetimerNrz%Could not retrieve modification time.rrs r1modifiedzWebHDFS.modifiedsOyy+T22  )%$,77 7BCCCr2c F|d|}|dd}|D];}|||ddz|dz|d<<|rt |d St d |DS) N LISTSTATUSr FileStatusesr/ pathSuffixrc|dS)Nrr)is r1zWebHDFS.ls..(s qyr2)keyc3&K|] }|dV dS)rNr).0rzs r1 zWebHDFS.ls..*s&994$v,999999r2)rir\r{rstripsorted)r'rbdetailr/reinfosrzs r1lsz WebHDFS.ls!sjjDj11 >*<8 G GD   t $ $ $;;s++c1D4FFDLL  :%%8%8999 999599999 9r2cd|d|}|dS)z8Total numbers of files, directories and bytes under pathGETCONTENTSUMMARYrContentSummaryrir\)r'rbres r1content_summaryzWebHDFS.content_summary,s,jj,4j88xxzz*++r2cp|d|d}d|jvrh||jd}|j|}||dS||dS)z/Checksum info of file, giving method and resultGETFILECHECKSUMF)rbrcLocation FileChecksum)riheadersrTr<rBrar\)r'rbrelocationout2s r1ukeyz WebHDFS.ukey1sjj*jFF  $ $((Z)@AAH<##H--D  ! ! # # #99;;~. .  " " "88::n- -r2c`|d}|dS)zGet user's home directoryGETHOMEDIRECTORYPathr)r'res r1home_directoryzWebHDFS.home_directory=s'jj+,,xxzz&!!r2c|r|d|}n|d}|d}|td|dS)zRetrieve token which can give the same authority to other uses Parameters ---------- renewer: str or None User who may use this token; if None, will be current user GETDELEGATIONTOKEN)renewerTokenNz1No token available for this user/security context urlString)rir\r )r'rrets r1get_delegation_tokenzWebHDFS.get_delegation_tokenBsf  3**17*CCCC**122C HHJJw  9PQQ Q~r2cf|dd|}|dS)z/Make token live longer. Returns new expiry timeRENEWDELEGATIONTOKENputrGr+longr)r'r+res r1renew_delegation_tokenzWebHDFS.renew_delegation_tokenSs-jj/UjKKxxzz&!!r2c6|dd|dS)z Stop the token from being usefulCANCELDELEGATIONTOKENrrNri)r'r+s r1cancel_delegation_tokenzWebHDFS.cancel_delegation_tokenXs! *5 FFFFFr2c8|dd||dS)aSet the permission at path Parameters ---------- path: str location to set (file or directory) mod: str or int posix epresentation or permission, give as oct string, e.g, '777' or 0o777 SETPERMISSIONr)rGrb permissionNr)r'rbmods r1chmodz WebHDFS.chmod\s$ ?5t LLLLLr2cFi}|||d<|||d<|jdd|d|dS)zChange owning user and/or groupNownergroupSETOWNERrrGrb)rr)r'rbrrr/s r1chownz WebHDFS.chownisJ  #F7O  #F7O Ae$AA&AAAAAr2c8|d|d|dS)a9 Set file replication factor Parameters ---------- path: str File location (not for directories) replication: int Number of copies of file on the cluster. Should be smaller than number of data nodes; normally 3 on most systems. SETREPLICATIONr)rbrGrpNr)r'rbrps r1set_replicationzWebHDFS.set_replicationrs% #$u+ VVVVVr2c 6|dd|dS)NMKDIRSrrrr'rbr/s r1mkdirz WebHDFS.mkdirs  8E 55555r2c|dur$||rt|||dS)NF)existsFileExistsErrorr)r'rbexist_oks r1makedirszWebHDFS.makedirssD u  T!2!2 !$'' ' 4r2c 8|dd||dS)NRENAMEr)rGrb destinationr)r'path1path2r/s r1mvz WebHDFS.mvs" 8E5 IIIIIr2c @|dd||rdnddS)NDELETEdeletetruefalse)rGrb recursiver)r'rbrr/s r1rmz WebHDFS.rms<  )6ffw      r2c 0||dSr4)rrs r1rm_filezWebHDFS.rm_files  r2c 8||5}d||dtjdg} ||d5}t j||dddn #1swxYwY|||nP#t$rCtt5| |dddn #1swxYwYwxYw ddddS#1swxYwYdS)Nrz.tmp.wb) openjoin_parentsecrets token_hexshutil copyfileobjr BaseExceptionrr^r)r'lpathrpathr/lstream tmp_fnamerstreams r1cp_filezWebHDFS.cp_files YYu   $,,u"5"57Vw?PQS?T?T7V7V!WXXI YYy$//97&w888999999999999999 5))))    /00''GGI&&&''''''''''''''' *                  srADB1-B B1B B1B B10D1C>C1% C>1C5 5C>8C5 9C>>DDDc|jr*t|jr||}n=|jr6|jD]\}}|||d}|S)N)rcallableitemsreplace)r'rkvs r1rTzWebHDFS._apply_proxysy : 5(4:.. 5zz(++HH Z 5 ((** 5 51#++Aq!44r2) rFNNNNNNFNT)rBNNT)rjNTNNFr4)NN))__name__ __module__ __qualname____doc__strtempfile gettempdirrnrrpropertyr6r%rirt staticmethodr{ classmethodrSrrzrrrrrrrrrrrrrrrrrrrT __classcell__r0s@r1r r s*c%(%''((G#H  c7c7c7c7c7c7JXHHH$H+ + + + Z\ 33[3\((( TTTDDD : : : :,,, . . .""" """" GGG M M MBBBB W W W666 JJJ       r2r cBeZdZdZfdZd dZdZdZdZdZ xZ S) rsz"A file living in HDFS over webHDFSc Htj||fi||}|dd|dd|dd|dd|dd|_|d}|dddurQ|j|_tj |ttj |_dSdS)NrqrpirnroF) rrrUrBrrqrbtargetosrr uuiduuid4)r'fsrbr/rnr0s r1rzWebHDFile.__init__sT,,V,,, ::mT * * 2 JJ}d + + + ::mT * * 2 JJ}d + + +!::mS99**Y'' ::lE * *e 3 3)DK Wc$*,,.?.?@@DIII 4 3r2Fc|jj|j|jddi}|dS)zWrite one part of a multi-block file upload Parameters ========== final: bool This is the last block, so should complete file, if self.autocommit is True. content-typeapplication/octet-stream)rIrT)rr<postrbuffergetvaluera)r'finalres r1 _upload_chunkzWebHDFile._upload_chunksZgo"" M%%''#%?@#   tr2c|j}d|jvrd\}}n d\}}d|d<|jj|||jfddi|}|j|jd}d |jvr|jj |d d i }| |jjd d|jfddi|}|j|jd|_ dSdS)zCreate remote file/uploada)APPENDPOST)CREATEPUTr overwritercFrwrr)rr"r#N) r/rUrlrrirbrTrr<rrar)r'r/rFrGrerrs r1_initiate_uploadzWebHDFile._initiate_uploads!!## $)  )JB(JB"(F; dgmB LLELVLL7'' J(?@@ $)  7?&&>3M"N'D  ! ! # # # 47=649WWuWPVWWD G00j1IJJDMMM  r2ct|d}t|j|}||ks ||jkrdS|jd|j|||z d}|d|jvrK|jd}|jj |j |}|j S|j S)Nrr2OPENF)rboffsetrwrcr) maxminrxrrirbrarr<rBrTcontent)r'startendrerrs r1 _fetch_rangezWebHDFile._fetch_rangesE1 $)S!! C<<5DI--3gmm 5uu     $ ${:.H7?&&tw';';H'E'EFFD< ; r2cP|j|j|jdSr4)rrrbrr5s r1commitzWebHDFile.commits"  49dk*****r2cD|j|jdSr4)rrrbr5s r1discardzWebHDFile.discards  49r2r) rrrrrrr(r1r3r5rrs@r1rsrss,, A A A A A"KKK( +++r2rs)loggingrrrr r contextlibrr urllib.parserr:specrrutilsr r getLoggerrXr rsrr2r1r<s1   ;;;;;;;;33333333  9 % %VVVVV VVVr IIIII$IIIIIr2