dtj6ddlZddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z ddl ZddlmZ ddlZn#e$r e sddlZYnwxYwddlmZddlmZddl mZmZmZdd lmZdd lmZmZm Z ej!d Z"Gd d e#Z$dZ%dZ&dZ'Gddej(j)Z*Gddej(j+Z,dZ-Gddej(j.Z/GddeZ0GddeZ1dS)N) lru_cache)chain) TYPE_CHECKINGLiteral)AbstractBufferedFile)AsyncFileSystem)DEFAULT_CALLBACK) filesystemopensplit_protocol)AsyncFileSystemWrapper) isfilelikemerge_offset_ranges other_pathszfsspec.referencec$eZdZfdZdZxZS)ReferenceNotReachablecPtj|||_||_dSN)super__init__ referencetarget)selfrrargs __class__s ^/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/fsspec/implementations/reference.pyrzReferenceNotReachable.__init__#s)$" c&d|jd|jS)Nz Reference "z" failed to fetch target )rrrs r__str__zReferenceNotReachable.__str__(sST^SSdkSSSr)__name__ __module__ __qualname__rr __classcell__rs@rrr"sN TTTTTTTrrc^tt|Sr)nextitervalues)ds r_firstr+,s QXXZZ  ! !!rc||}t|ttfrFt|dtr-|drt |ddn|dSdSdSNr)get isinstancelisttuplestrr )path referencesrefs r_prot_in_referencesr60s{ ..  C#e}%%?*SVS*A*A?,/F>~c!f%%a((A>????rct|trt|||giSi}|D];}t||}||g|<|Sr)r/r2r6 setdefaultappend)pathsr4outr3protocols r_protocol_groupsr=6sy%A#E:66@@ C22&tZ88 x$$++D1111 JrceZdZdZdS)RefsValuesViewc#K|jjD]*}tj|V+|jjEd{V|jD]b}|j|}t|dkr|j|dzVB|j |Ed{VcdS)Nr/0) _mapping zmetadatar)jsondumpsencode_itemslistdir_get_chunk_sizeslen_generate_all_records)rvalfield chunk_sizess r__iter__zRefsValuesView.__iter__As =*1133 + +C*S//((** * * * *='..000000000]**,, B BE-88??K;1$$mEDL1111}::5AA A A A A A A A A  B BrNr!r"r#rOrrr?r?@s( B B B B Brr?ceZdZdZdS) RefsItemsViewc~t|j|jSr)ziprBkeysr)rs rrOzRefsItemsView.__iter__Ns.4=%%'')=)=)?)?@@@rNrPrQrrrSrSMs(AAAAArrScxd}d}t|ddd|dddD]\}}|||zz }||z}|S)Nr)rU)idxsizesrLmultiss rravel_multi_indexr_RsZ C DC"IuTTrT{++1 q4x   JrceZdZdZedZedZ d%ded fd Zd Z d Z e d&dZ dZ d'dZdZdZdZdZdZdZdZdZdZdZe dZdZd(dZd(d Zd!Zd"Zd#Zd$Z dS))LazyReferenceMapperaoThis interface can be used to read/write references from Parquet stores. It is not intended for other types of references. It can be used with Kerchunk's MultiZarrToZarr method to combine references into a parquet store. Examples of this use-case can be found here: https://fsspec.github.io/kerchunk/advanced.html?highlight=parquet#parquet-storagecddl}|Sr-)numpy)rnps rrdzLazyReferenceMapper.npds rcddl}|Sr-)pandas)rpds rrgzLazyReferenceMapper.pdjs rN fastparquetengine)rjpyarrowc||_i|_||_||_||_|jdz|_|t jdn||_|j |p|j|_ ddl m }|jdkr|dtdt|j|_td |j|_dS) a This instance will be writable, storing changes in memory until full partitions are accumulated or .flush() is called. To create an empty lazy store, use .create() Parameters ---------- root : str Root of parquet store fs : fsspec.AbstractFileSystem fsspec filesystem object, default is local filesystem. cache_size : int, default=128 Maximum size of LRU cache, where cache_size*record_size denotes the total number of references that can be loaded in memory at once. categorical_threshold : int Encode urls as pandas.Categorical to reduce memory footprint if the ratio of the number of unique urls to total number of refs for each variable is greater than or equal to this number. (default 10) engine: Literal["fastparquet","pyarrow"] Engine choice for reading parquet files. (default is "fastparquet") z/{field}/refs.{record}.parqNfiler) find_specrlz)engine choice `pyarrow` is not installed.imaxsize)rootrN cat_threshrk cache_sizeurlfsspecr fsunstrip_protocolout_rootimportlib.utilro ImportErrorrrH_key_to_record)rrrrwryrtcategorical_thresholdrkros rrzLazyReferenceMapper.__init__psB / $9<</1z&#F+++r001FTYGG ,,,,,, ;) # # )(<(<(DIJJ J #y{{4<00 5i555d6IJJrcj|dvr!||j|St|)N)rG record_sizerC)setup__dict__AttributeErrorritems r __getattr__zLazyReferenceMapper.__getattr__s6 9 9 9 JJLLL=& &T"""rcNi_jdjdgjd<t jjd}|d_|d_tj fd}|_ dS)N/ .zmetadatarmetadatarpc*j||}tjj|} j|jfdj D}n#t$rd}YnwxYw|S)zcached parquet file loader)rMrecord)rkcJi|]}||d S)Tcopy)to_numpy).0cdfs r z@LazyReferenceMapper.setup..open_refs..s-III2a5>>t>44IIIrN) ruformatioBytesIOrwcat_filerg read_parquetrkcolumnsOSError)rMrr3datarefsrrs @r open_refsz,LazyReferenceMapper.setup..open_refss8??v?>>D:dg..t4455D W))$t{)CCIIIIbjIII    Ks 4B BB) rGrwrjoinrrrDloadsrrCrrtr)rmetrs` rrzLazyReferenceMapper.setups $(G$4$4 HHdi. / /% %  L!j\233}-Z 4? + + +     , + #r'c i|d}|tjj|fi|pi\}}||r||d||d|d|dgtj | t||fi|S)aJMake empty parquet reference set First deletes the contents of the given directory, if it exists. Parameters ---------- root: str Directory to contain the output; will be created storage_options: dict | None For making the filesystem to use for writing is fs is None fs: FileSystem | None Filesystem for writing record_size: int Number of references per parquet file kwargs: passed to __init__ Returns ------- LazyReferenceMapper instance rrNT recursiveexist_okrr) rvcore url_to_fsexistsrmmakedirspiperrDrErFra)rrstorage_optionsrwrkwargsrs rcreatezLazyReferenceMapper.creates,k:: :{,TMMo6KMMHB 99T?? ( EE$$E ' ' ' D4 ((( $ -.. 30F0F0H0HIII"466v666rcBd|jD}t|S)zList top-level directoriesc3xK|]5}|d|dddV6dS).zrrXrN) startswithrsplitrps r z.LazyReferenceMapper.listdir..sGVV1<z*LazyReferenceMapper.ls..sP   ||H%% +.QXXh5G5G*G*G *G*G*Grclh|]0}|rd|v.|1Srr)rfrs r z)LazyReferenceMapper.ls..sU <<))/2(9K9K.K.K  /L.K.Krrc3K|]}|VdSrrQrnames rrz)LazyReferenceMapper.ls..s"55dT555555rc3K|]}|VdSrrQrs rrz)LazyReferenceMapper.ls..s"22dT222222rFcg|]}|ddd S) directoryrrtypesizerQrs rrz*LazyReferenceMapper.ls..s"WWW$k1EEWWWrc g|]H}|dt|jvrtjj|n j|dISrnrrJrCrDrErGrrrs rrz*LazyReferenceMapper.ls..su   !"4>11 4>$#7888![.   rc|dS)NrrQ)r^s rz(LazyReferenceMapper.ls..s AfIrkeycDg|]}|d|Srrrrr3s rrz*LazyReferenceMapper.ls..s0 L L Ld4 0K0K LT L L LrcDg|]}|d|Srrrs rrz*LazyReferenceMapper.ls..s0KKKt$zzz/J/JKtKKKrc g|]H}|dt|jvrtjj|n j|dISrrrs rrz*LazyReferenceMapper.ls..su    t~--Jt~d3444T*      rc<g|]\}}|d |d|ddS)rrnrYrrQ)rrrecs rrz*LazyReferenceMapper.ls..%sD   c1v 63r7 ; ;   r) rstriprHrrCrGupdatesortedr_keys_in_fieldr0rKrU) rr3detaildirnamesrothersdirinfofileinforMrVrecsrecinfors `` @rlszLazyReferenceMapper.lss;{{3!%-4#::2<<>>         G!N55dn55522dk222F d###f~~%WWRVWWWG    #   H'H,2E2EFFF F L L L Ldn L L LKKKK KKK L           ""5)) U??<<$t**, ,))%00   t__    '!!rc( ||jvr |j|S||jvr1tj|j|Sd|vs||rt ||dd\}}||\} }|j ||fi d}|t |r|S|dkrdS | || n/#tttf$r}t ||d}~wwxYwgd} fd|D} | d } | | S| dt d | dd ddgkr | ddS| dd S) zfGet the reference for one key Returns bytes, one-element list or three-element list. rrXFNrrr3offsetrrawc8g|]}|vr|ndSrrQ)rrrris rrz5LazyReferenceMapper._load_one_key..Hs-IIIAAIIT!WR[[4IIIrrYz1This reference does not exist or has been deleted)rGrCrDrErF_is_metaKeyErrorrr|r.r ValueError TypeErrorFileNotFoundError) rrrM_r chunk_sizemaybeexcr selectionrrrs @@r _load_one_keyz!LazyReferenceMapper._load_one_key,s $+  ;s# # DN " ":dnS12299;; ; ^^t}}S11^3-- ::c1%%q!%!4!4S!9!9J 4488UCC =N  L 1__3 )>>%00DDI'89 ) ) )3--S ( )333IIIIIIII m ?J Q< NOO O QqS>aV # #RaR= !}s4D D7"D22D7c2|dd\}}||}t|dkrdSd|dD}t ||}||jz}||jz}||t|fS)z3Details needed to construct a reference for one keyrrXr)rrrc,g|]}t|SrQ)intrrs rrz6LazyReferenceMapper._key_to_record..Zs666SVV666r.)rrIrJsplitr_r) rrrMchunkrN chunk_idx chunk_numberrrs rr|z"LazyReferenceMapper._key_to_recordTszz#q)) u++E22 {  q 766U[[%5%5666 (K@@ !11 D, ,r3{++++rc||jvrC|j|d}dt|d|dD}|pdg|j|<|j|S)z6The number of chunks along each axis for a given fieldz/.zarraycBg|]\}}tj||z SrQ)mathceil)rr^rs rrz8LazyReferenceMapper._get_chunk_sizes..ds7%)Q !a%  rshapechunksrX)rNrCrU)rrMzarray size_ratios rrIz$LazyReferenceMapper._get_chunk_sizes`sz ( ( (^u$6$6$67F-0&BR-S-SJ'1&7QCD U #&&rc |||}tt|}t |dkr d|DSt |dkr|dSd|DS)z8The references for a given parquet file of a given fieldrc34K|]}t|VdSrr0rts rrz7LazyReferenceMapper._generate_record..ps(((DGG((((((rrXrc3dK|]+}|dst|ddn|dV,dS)rNrrs rrz7LazyReferenceMapper._generate_record..vsABB!qt5D2A2KKK1BBBBBBr)rr(rUr)rJ)rrMrrits r_generate_recordz$LazyReferenceMapper._generate_recordjs~~eV,, #t{{}}% & & t99>>((R((( ( YY!^^; CBrBBB Brc#Kd}||D]}||z}tj||jz }t |D]}|||Ed{VdS)zJLoad all the references within a field by iterating over the parquet filesrXN)rIrrrranger)rrMnrecchrs rrKz)LazyReferenceMapper._generate_all_recordsxs''..  B BJDDy 0011Dkk < >D, , , >>%88      @V$Ex(GV$EE?DDGGD,bfCG@@Ehht/wh??GHHT-WH==E774+RV37??D (( 6 6GAt$%% 6 $$ 22Qu{'===!00a99E7at99q==!%aGAJ#AwE!Ha aQ#+//55Q \\!       7==??bgoo//41 5 G GF**:66BvJ")6::UO )DM F FE F F F F F (3dm==e==MMM ;) # #!3U ;   [M ) )#2&!!   &&D&D&DEE E  ;+9tw 1488         (((((sA AAct|jD]5}t|tr|\}}|||||6t|jD]@}|dkr8d|vr4t j|j||j|<A|j|j d}|j t j | |jd<|j d|p|jdg|jd|jdS)zOutput any modified or deleted keys Parameters ---------- base_url: str Location of the output )rMrrrrrN)r0rGr/r1r#rDrrLrCrrKrErFrwrrryr cache_clear)rrMrthingrMrkrs rflushzLazyReferenceMapper.flush sT$+&&  E%''  % v %$3 dk"" C CAL  TQYY$(Jt{q/A/A$B$Bq!>$:JKK $(JsOO$:$:$<$< L!  HHh/$-> ? ? K %   ""$$$$$rc2d}|D]G}|dr|dz }|tj||z }H|t |jz }|td|jDz }|S)NrrrXc3DK|]}t|tdVdS)rXN)r/r1)rrs rrz.LazyReferenceMapper.__len__..8s1HH1:a3G3GHQHHHHHHr) rHrrprodrIrJrCsumrG)rrDrMs r__len__zLazyReferenceMapper.__len__.s\\^^ A AE$$ A 4#8#8#?#?@@@ T^$$$ HH HHHHHH rc#Kt|j}||j|D]}t |t r|V|D]"}||D] }||vr|V #dSr)rrCrrGr/r2rHr)rmetasbitrMrYs rrOzLazyReferenceMapper.__iter__;sDN## T[!!!  C#s##  \\^^  E((//  99GGG   rcT ||dS#t$rYdSwxYw)NTF)rrrs r __contains__z LazyReferenceMapper.__contains__HsB    t $ $ $4   55 s  ''c#K||}t|dkr |dzVdStjd|D}|D])}|dzdd|DzV*dS)z|List key names in given field Produces strings like "field/x.y" appropriate from the chunking of the array rrANc34K|]}t|VdSr)r)rr]s rrz5LazyReferenceMapper._keys_in_field..Xs("A"A588"A"A"A"A"A"Arrrc,g|]}t|SrQ)r2rs rrz6LazyReferenceMapper._keys_in_field..Zs)>)>)>Q#a&&)>)>)>r)rIrJ itertoolsproductr)rrMrNindsinds rrz"LazyReferenceMapper._keys_in_fieldOs ++E22 {  q $,    F "A"A["A"A"AB @ @C#+)>)>#)>)>)> ? ?? ? ? ? ? @ @r)NNrhrirj)NNr)rTNN)!r!r"r#__doc__propertyrdrgrrrr staticmethodrrHrrr|rIrrKr)rrr r*rr-r#rZr_rOrdrrQrrrara[s3YYX X  4A5K5K015K5K5K5Kn######0777\7< @"@"@"@"D&&&P , , ,''' C C C<<<$$$###'''OOO(44\4((("S)S)S)S)j!%!%!%!%F       @ @ @ @ @rraceZdZdZdZdZ d(fd Zd)d Zd)d Zd)d Z d Z dZ e fdZ d*dZd+dZd,dZdZd,dZdZdZdZd-dZd.dZdZdZdZd.dZd/fd Zd!Zd"Zd#Zd0d%Z d0d&Z!d'Z"xZ#S)1ReferenceFileSystemaCView byte ranges of some other file as a file system Initial version: single file system target, which must support async, and must allow start and end args in _cat_file. Later versions may allow multiple arbitrary URLs for the targets. This FileSystem is read-only. It is designed to be used with async targets (for now). We do not get original file details from the target FS. Configuration is by passing a dict of references at init, or a URL to a JSON file containing the same; this dict can also contain concrete data for some set of paths. Reference dict format: {path0: bytes_data, path1: (target_url, offset, size)} https://github.com/fsspec/kerchunk/blob/main/README.md simple_references: if True (default), no jinja interpreting is done, which is the safe option. rFNT@Brhc tjdi|||_| |_| |_i|_i|_i|_| |_| |_ t|trtdi|p|pid|i}tjj|fi|\}}d|vr]|ds||r3t$d|t)||| |_ntj|dfi|5}t$d|t/j|}dddn #1swxYwY||| n||| t|trEd |D|_d|jvrt7d |jd<dS|8t|jt:r |jd n|j}||j|<|{|jD]a}t?|r |}tj |\}}|r"||jvrt7|fi|pi}||j|<b||jD]}t?|r |}t|tBrT|d rLtj |d \}}||jvrt7|fi|pi}||j|<n|r"||jvrt7|fi|pi}||j|<|pt7d |jd<|jD]I\}}|j"stG||j$ |j|<+|j$|j$z rtKd JdS)a Parameters ---------- fo : dict or str The set of references to use for this instance, with a structure as above. If str referencing a JSON file, will use fsspec.open, in conjunction with target_options and target_protocol to open and parse JSON at this location. If a directory, then assume references are a set of parquet files to be loaded lazily. target : str For any references having target_url as None, this is the default file target to use ref_storage_args : dict If references is a str, use these kwargs for loading the JSON file. Deprecated: use target_options instead. target_protocol : str Used for loading the reference file, if it is a path. If None, protocol will be derived from the given path target_options : dict Extra FS options for loading the reference file ``fo``, if given as a path remote_protocol : str The protocol of the filesystem on which the references will be evaluated (unless fs is provided). If not given, will be derived from the first URL that has a protocol in the templates or in the references, in that order. remote_options : dict kwargs to go with remote_protocol fs : AbstractFileSystem | dict(str, (AbstractFileSystem | dict)) Directly provide a file system(s): - a single filesystem instance - a dict of protocol:filesystem, where each value is either a filesystem instance, or a dict of kwargs that can be used to create in instance for the given protocol If this is given, remote_options and remote_protocol are ignored. template_overrides : dict Swap out any templates in the references file with these - useful for testing. simple_templates: bool Whether templates can be processed with simple replace (True) or if jinja is needed (False, much slower). All reference sets produced by ``kerchunk`` are simple in this sense, but the spec allows for complex. max_gap, max_block: int For merging multiple concurrent requests to the same remote file. Neighboring byte ranges will only be merged when their inter-range gap is <= ``max_gap``. Default is 64KB. Set to 0 to only merge when it requires no extra bytes. Pass a negative number to disable merging, appropriate for local target files. Neighboring byte ranges will only be merged when the size of the aggregated range is <= ``max_block``. Default is 256MB. cache_size : int Maximum size of LRU cache, where cache_size*record_size denotes the total number of references that can be loaded in memory at once. Only used for lazily loaded references. kwargs : passed to parent class r<z.json)parqparquetrz$Open lazy reference dict from URL %s)rwrtrbzRead reference from URL %sNc i|]I\}}|t|tr,tj|dddfi|n|JS):rXr)r/dictrvr r)rrYoptss rrz0ReferenceFileSystem.__init__..sl At !$--F%aggc1ooa&8AADAAA rrnr) asynchronouszEReference-FS's target filesystem must have same value of asynchronousrQ)&rrrtemplate_overridessimple_templates templatesfss _dircachemax_gap max_blockr/r2rzrvrrendswithisdirloggerinforar4r rDload_process_referencesrr r<r1r)callabler r0 async_implr r|r)rforref_storage_argstarget_protocoltarget_optionsremote_protocolremote_optionsrwr}r~rrrtrdicref_fsfo2rtextr5r<rrYrs rrzReferenceFileSystem.__init__rsR ""6""" "4 0 " b#   =#;~;GVC!+/::c::KFCc!! 455"9?c9J9J" BBGGG"5)###[T11S11(QKK #-R[%"@"@Q Abk )+DH_ %  "~,,.. , ,C== #%%C$k88== !, 8 8#HGG1E2GGB)+DHX&  "--//  C== #%%Cc4((SV"(+"<">OFF0D"FFB(*DH_ %1z&11HNN$$  DAq< 4QTEVWWW "Q^3  &   s 0EE E c ||}td| |j|}n"#t$r}t ||d}~wwxYwt |tr|}t|dr| }t |trTtd|d| drtj|dd}|ddfSt|dkr-td|d||d }||}}n]|\}} } td|d |d | d | | | z} ||d kr| |z}n| |z}n| }||d kr| |z}n| |z}n| }||j}|||fS) Nzcat: r"z Reference: z , type bytesbase64:rXz, whole file => rz => z , offset z, size )_strip_protocolrdebugr4rrr/r2rFr$r"r&rbase64 b64decoderJr) rr3startendpartrrustart1end1start0rend0s r _cat_commonzReferenceFileSystem._cat_commons##D)) ^T^^$$$ 3?4(DD 3 3 3#D))s 2 3 dC  !;;==D 4 $ $ #==??D dE " " $ LL9t999 : : :z** 2'QRR11t# # t99>> LLCtCCTCC D D Dq'C #DFF $ C LLTtTTTTvTTdTT U U UD=D A::#e^FF!E\FF!88!Cg|]}||SrQ)r)rrrs rrz+ReferenceFileSystem.get..vs(CCC1TZZ]]CQCCCr)r expand_pathrvr rcatrUr) rrrrrrwtargetsrremotelocals ` rr.zReferenceFileSystem.getns   GGBKKK  ) <<  v$ 7 7 7eU++  #88CCCCCCCDDDD88E??D 00 2 2MFE~~ UDL111 2 2rraisec t|tr |rtt|tr"|st d|Drtt ||j}i}|D]\}}|j|} ggggf\} } } } |D]} | |\}}}t|ttfs9 | || || || |#t$r}|dkr|dkr|||<Yd}~d}~wwxYwg}g}g}g}t}t| | | | D]\}}}}t|tr|||<"|i||| || || || |t| | | | D]a\}}}}|X||vrT| || || || |bt!t|t|t|d|j|j\}}}| |||}t| | | | D]\}}}}||vr t||||D]|\}}}}||kr-||)t|t(r|||<,|||||<:||kr<||kr6||kr0t|t(r|||<g|||z ||z pd||<}|D]`\} }!t|!t(rF| |jvr=|| }"t-| |j| }#|"|#_|dkr|#|dkr|#|| <at1|dkr(t|trd|vrt3|S|S)Nc3K|]}d|vV dS)*NrQrs rrz*ReferenceFileSystem.cat..s&7O7OQq7O7O7O7O7O7OrromitT)sortrrrXr)r/r2rHr0anyr=r4rrrr&r9rrrUaddrrr cat_rangesrrr __cause__rJr+)$rr3ron_errorr proto_dictr;protor:rwurlsstartsends valid_pathsrur^rerrurls2starts2ends2paths2 whole_files new_paths new_startsnew_ends bytes_outrdnsnebrYvexnew_exs$ rrzReferenceFileSystem.cat}s dC  &Y &% % dD ! ! &y &C7O7O$7O7O7O4O4O &% %%dDO<< &,,..I BI BLE5%B.0"b"n +D&$  * **"..q11GAq!%a%66! !KKNNNMM!$$$KKNNN&&q)))))%%%7**6))!$A %EGEF%%K!$kBB % % 1aa'' %CFFY OOA&&&LLOOONN1%%%LLOOOMM!$$$!$kBB % % 1a=Qk%9%9LLOOONN1%%%LLOOOMM!$$$/BU W U  . /// +Iz8 iXFFI"$kBB B B 1a88%(J)%T%T B BMBBQwwBJ"*%a33,%&CFF%&qsVCFFqQ"WWb%a33B%&CFF%&q2vR0@D'@%ACF B BHHJJ$$&& $ $DAq!Y'' $A,@,@V.q$/!2DEE#% w&& L''#CF s88q==Zc22=s$#;;  s$5D00 E:EEc|dd}|||dS|dkr|||dStd|)NversionrX)r}z Unknown reference spec version: )r._process_references0_process_references1r)rr4r}verss rrz'ReferenceFileSystem._process_referencesst~~i.. <  % %j 1 1 1 1 1 QYY  % %jEW % X X X X XFFFGG Grczt|trd|D}||_dS)z&Make reference dict for Spec Version 0cli|]1\}}|t|trtj|n|2SrQ)r/rzrDrE)rrrLs rrz.sIC 3(=(=FTZ___3rN)r/rzrr4)rr4s rrz(ReferenceFileSystem._process_references0sK j$ ' '  * 0 0 2 2J%rcjrjrddli_|dit dfd}|diD],\}}t|trD| dr$tj |ddj|<|j|<_t|trtj|j|<jr|d}d|vrMjr;|dd d d jdij}n ||}t%|d kr|gn||d |d gj|<"|j|<.j|dgdS)Nrric|j|jdijSNrQ)sandboxSandboxedEnvironment from_stringrenderr)rjinja2rs r _render_jinjaz?ReferenceFileSystem._process_references1.._render_jinjasD3355Q**.** rrzbase64:r{{{z}}}rXr3genrQ)r~rjinja2.sandboxr4_process_templatesr.rrr/r2rrrrzrDrEreplacerrJr _process_gen)rr4r}rrYrrrs` @rrz(ReferenceFileSystem._process_references1s$ " " ! ! ! !  {B ? ?@@@ 4       NN62..4466 ' 'DAq!S!! '<< **A)/)9!ABB%)@)@DOA&%&""At$$ '%)Z]]"" 'aD199,-$AIIdC00$WT3//#66&*n66 *M!,,,/FFaKKaSSa1qt_""%&"" t00r1J1JKKLLLLLrci|_|j||j|D]'\}}d|vrddl|ffd |j|<||j|<(dS)Nrrcrj|jdi|Sr)rrrr)temprrs rrz8ReferenceFileSystem._process_templates..#sA.V^-P-P-R-R [&&.&.&$.&.&r)rr}rrr)rtmprYrrs @rrz&ReferenceFileSystem._process_templatess  " . JJt. / / /IIKK & &DAqqyy%%%%!"&&&&&q!! %&q!! & &rc  i}|jr|S|D]}d|dD fdtj D}|D]a}ddl}|j|dj d i||j }|j|dj d i||j }d|vrd|vrt|j|dj d i||j } t|j|dj d i||j } || | g||<Cd|vd|vz rtd |g||<c|S) Nc i|]\\}}|t|tr|n>t|dd|d|dd]S)rrstopsteprX)r/r0rr.)rrYrs rrz4ReferenceFileSystem._process_gen../st Aq !!T**OAAquuWa00!F)QUU61=M=MNN r dimensionsc3vK|]3}tt|V4dSr)rzrUrV)rr) dimensions rrz3ReferenceFileSystem._process_gen..7sSS))62233rrrrurlengthz^Both 'offset' and 'length' are required for a reference generator entry if either is provided.rQ) r~rrhrir)rrrrrrrr) rgensr;rproductsprrrrurr r s @rrz ReferenceFileSystem._process_gen*ss   J, %, %C  -3355 I'/1A1A1C1CDH % %%%%%FN7799 [U,,44 4$(N44 FN7799 [U,,44 4$(N44 OO(c//  ;;==$S]3388"$8(,88F ! ;;==$S]3388"$8(,88F !$VV4CHH#o(c/:%$K !$uCHH? %@ rcdgi|_|j}|D]_\}}t|tt fst |drt|}nt|dkrd}n|\}}}d|vr|dddnd}|}|g}|r>||jvr5| |}| ||r ||jv5| t||ddD]I\} } | |jvsJ| |jvsJ|j|  | dddg|j| <J|j| |d|dadS) Nrr"rXrrrrrn) dircacher4rr/r&r2r$rJr_parentr9reverserU) rrr3rrrparpar0subdirsparentchilds r_dircache_from_itemsz(ReferenceFileSystem._dircache_from_items]sR _ " " $ $ T TJD$$ -- "z1J1J "4yyTa! 1d,/4KK$++c1%%a((RCDfG %4t}44||D))t$$$ %4t}44 OO   !$Wgabbk!:!: * * DM1111.... f%,,"KCC(* e$$ M#  % %tVT&R&R S S S S3 T Trrwc ||\}}}t|trtj|||St |\} } |||j| j||f||d|St|||f||d|S)N) block_size cache_options) rr/r&rrr r_open ReferenceFile) rr3moderrrrrrr<rs rrzReferenceFileSystem._open{s$($4$4T$:$:! VT k5 ) ) 8:k&+677 7$[11 ! >dl+48H%+&+        "'       rc td|||}t|jt r? |j||S#t$rYnwxYwtd|d|j s| | |}|t||r|Sd|DS)Nzlist %s'z' is not a known keycg|] }|d SrrQ)ros rrz*ReferenceFileSystem.ls..s'''a& '''r) rrrr/r4rarrrrr_ls_from_cache)rr3rrr;s rrzReferenceFileSystem.lss Y%%%##D)) do': ; ; D ))$777    #$B$B$B$BCC C} (  % % ' ' '!!$'' ;#D)) )  J''3''''s A'' A43A4c V||p||Sr)risfilerr3rs rrzReferenceFileSystem.existss%zz$44;;t#4#44rc|jr |jvSt|jtr|jvSt fd|jDS)Nc3HK|]}|dVdS)rNr)rrr3s rrz,ReferenceFileSystem.isdir..s5IIAq||tJJJ//IIIIIIr)rr/r4rarHrrr3s `rrzReferenceFileSystem.isdirsq = J4=( ( )< = = J4?22444 4IIIIIIIII Irc||jvSrrr)s rr%zReferenceFileSystem.isfilest&&rc$K|j||fi|Sr)r)rr3rrs r_lszReferenceFileSystem._lss"twtV..v...rc :|rtjf|||d|Sr6tfdjD}ntj}|r)jsfd|DS|S)N)maxdepthwithdirsrc3FK|]}||VdSrr)rrYr3s rrz+ReferenceFileSystem.find..s4HHQQ\\$5G5GHqHHHHHHrcHi|]}||dS)r)r#)rrYrs rrz,ReferenceFileSystem.find..s.<<.s'888!ai4&7&7A&7&7&7rrrr) r4r.r/r2r&rJrr rr)rr3rr;out0protrs ` rrzReferenceFileSystem.infos'o!!$'' ?#U|,, F $fc#hhGGGSA $fc!fEEE!%vtDDE''$%%C8888s888D F $k1EEE 76? "$T_T%:1%=>>GD!"htn11$/$2G2JKKDGFOAwrc0K||Sr)rr&s r_infozReferenceFileSystem._infosyyrcrK|j|d|jdSr)r4rLrrKr&s r_rm_filezReferenceFileSystem._rm_filesA  $    r overwritecK|dkr||rt||j|<|jdS)Nr)rFileExistsErrorr4rrK)rr3rrrs r _pipe_filezReferenceFileSystem._pipe_filesN 8   D 1 1 ! ! $ rcK|dkr||rtt|d5}||j|<dddn #1swxYwY|jdS)Nrrw)rr=r readr4rrK)rrrrrrs r _put_filezReferenceFileSystem._put_files 8   E 2 2 ! ! %   .!%&VVXXDOE " . . . . . . . . . . . . . . . sAA"%A"c i}|jD]u\}}t|trV |d||<4#t $r/dt j|z||<YlwxYw|||<vtj |dfi|5}| tj d|d ddddS#1swxYwYdS)z+Write modified references into new locationasciirrrX)rrN)r4rr/r&r%UnicodeDecodeErrorr b64encodervr r#rDrErF)rrurr;rYrrs r save_jsonzReferenceFileSystem.save_jsonsjO))++  DAq!U## IXXg..CFF)III(6+;A+>+>>FFHHCFFFIA [d 6 6o 6 6 F! GGDJ1c::;;BBDD E E E F F F F F F F F F F F F F F F F F Fs#A6B B %=C//C36C3) NNNNNNNNTrrrsrhrl)F)Frr)rwNN)T)NFF)r;)$r!r"r#rmr<cachablerrrrrrr rr.rrrrrrrrrrrr%r,r2rr8r:r>rArFr$r%s@rrqrq]s"HH  aaaaaaF*!*!*!*!X B B B BBBBB&&&/? , , , , 2 2 2 2````DHHHH % % %#M#M#M#MJ&&& 111fTTT<    4(((($555JJJ'''////"*  F F F F F F FrrqcReZdZ d fd ZedZfdZd ZxZS) rrwdefaultT readaheadNc tj||f||||||d| |j|j\} |_|_t| \} } |jj| |_ | |_ d|_ dS)N)rr autocommitr cache_typer) rrrwrr3rrr rsrc_fssrc_path_f)rrwr3rrrLrMrrrrr<rrs rrzReferenceFile.__init__s    !!!'  -1G,?,? ,J,J) TZ$[11 !gk(+ # rc|j |jjr6|jj|jf|j|j|jdd|j|_|jS)Nnone)rrrLrM) rPclosedrNrrOr blocksizerLrrs rrzReferenceFile.f%s_ 7?dgn?'dk' Y>?!  + DGwrc|j|jtSr)rPcloser)rrs rrVzReferenceFile.close2s, 7  GMMOOOww}}rc||jz}t||jz|j}|j||j||z Sr)rminrrseekr@)rrrs r _fetch_rangezReferenceFile._fetch_range7sR "# "DH--  Ev{{3;'''r)rwrITrJNN) r!r"r#rrnrrVrZr$r%s@rrrs  :  X  (((((((rr)2r collectionsrrhloggingrr functoolsrrtypingrr fsspec.corerv fsspec.specrujsonrDr{ fsspec.asynrfsspec.callbacksr r r r #fsspec.implementations.asyn_wrapperr fsspec.utilsrrr getLoggerr RuntimeErrorrr+r6r=abc ValuesViewr? ItemsViewrSr_MutableMappingrarqrrQrrrls   )))))))),,,,,,  ('''''------8888888888FFFFFF  - . .TTTTTLTTT"""???  B B B B B[_/ B B BAAAAAKO-AAA @@@@@+/8@@@Dg Fg Fg Fg Fg F/g Fg Fg FT4(4(4(4(4((4(4(4(4(4(sA AA