:j lUddlmZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z m Z ddlmZddlmZddlmZmZmZmZmZmZerddlZdd lmZed Zned Zed Zej<d Ze e e ge!fZ"e e#e e fge!fZ$Gd dZ%Gdde%Z&Gdde%Z'Gdde%Z(Gdde%Z)Gdde%Z*Gdde%Z+Gdde%Z,GddeeefZ-Gdd e%Z.de%iZ/d!e0d"<d$d%d#Z1e%e&e*e'e)e(e+e,e.f D] Z2e1e2 y)&) annotationsN) OrderedDict)Callable)FutureThreadPoolExecutor)groupby) itemgetter) TYPE_CHECKINGAnyClassVarGeneric NamedTupleTypeVar) ParamSpecPTzfsspec.cachingcHeZdZUdZdZded<d dZd dZd dZddZ dd Z y ) BaseCacheagPass-though cache: doesn't keep anything, calls every time Acts as base class for other cachers Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is none ClassVar[str]namecf||_d|_||_||_d|_d|_d|_yNr) blocksizenblocksfetchersize hit_count miss_counttotal_requested_bytes)selfrrrs c/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/fsspec/caching.py__init__zBaseCache.__init__4s4"   %&"ct|d}| |j}||jk\s||k\ry|j||S)Nrr$)rrr!startstops r"_fetchzBaseCache._fetch>s@ =E <99D DII $||E4((r$c.d|_d|_d|_y)zAReset hit and miss counts for a more ganular report e.g. by file.rN)rrr r!s r" _reset_statszBaseCache._reset_statsGs%&"r$c |jdk(r|jdk(ryd|jd|jd|jd|jd S)z2Return a formatted string of the cache statistics.rz , z: z hits, z misses, z total requested bytes)rrrr r+s r" _log_statszBaseCache._log_statsMs^ >>Q 4??a#7TYYKr$..!18ISWSmSmRnoEF Fr$cd|jjd|jd|jd|jd|j d|j d|jdS) Nz ) __class____name__rrrrrr r+s r"__repr__zBaseCache.__repr__Ts .. ! ! "#!^^,-!\\N+!YYK(!^^,-!__-.$$($>$>#?@   r$NrintrFetcherrr5returnNoner' int | Noner(r:r7bytesr7r8)r7str) r2 __module__ __qualname____doc__r__annotations__r#r)r,r/r3r$r"rr"s- !D- ')' F  r$rcjeZdZdZdZ d d fd Zd dZd dZd dZd dZ xZ S) MMapCacheamemory-mapped sparse file cache Opens temporary file, which is filled blocks-wise when data is requested. Ensure there is enough disc space in the temporary location. This cache method might only work on posix Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: Fetcher Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is location: str Where to create the temporary file. If None, a temporary file is created using tempfile.TemporaryFile(). blocks: set[int] Set of block numbers that have already been fetched. If None, an empty set is created. multi_fetcher: MultiFetcher Function of the form f([(start, end)]) which gets bytes from remote as specified. This function is used to fetch multiple blocks at once. If not specified, the fetcher function is used instead. mmapct||||| tn||_||_||_|j |_yN)superr#setblockslocation multi_fetcher _makefilecache)r!rrrrKrJrLr1s r"r#zMMapCache.__init__sB GT2%~ce6   *^^% r$c4ddl}ddl}|jdk(r tS|j)t j j|js|j |j}t|_ nt|jd}|j|jdz |jd|jnt|jd}|j|j|jS)Nrzwb+1zr+b)rEtempfiler bytearrayrKospathexists TemporaryFilerIrJopenseekwriteflushfileno)r!rErRfds r"rMzMMapCache._makefiles 99>;  == t}}(E}}$++-!e $--/ GGDIIM " HHTN HHJdmmU+Btyydii00r$c  tjd|d||d}| j}|jk\s||k\ry|jz}|jz}t ||dz}fd|D}xj t fd|Dz c_g}tt|d D]\}} tttd| } | djz} t| d jzjzj} xj| | z z c_ tjd | dd| d d | d| d |j| | fjj!| xj"t%| z c_|sj&||Sj(rmtjd |tj)|D]7\} } || \} } tjd| d| | j&| | 9nD|D]?\} } tjd| d| j+| | j&| | Aj&||S)NzMMap cache fetching -rr$rPc3@K|]}|jvs|ywrGrJ.0ir!s r" z#MMapCache._fetch..s?a!4;;*>?c3@K|]}|jvsdyw)rPNrarbs r"rez#MMapCache._fetch..sIAT[[8HaIrfc|d|dz S)NrrPrB)xs r"z"MMapCache._fetch..s1!r$)keyzMMap get blocks z ()zMMap copy block (zMMap get block ()loggerdebugrrrangersumr enumeratetuplemapr minr appendrJupdaterlenrNrLr)r!r'end start_block end_block block_rangeneedranges__blockssstartsendidxrs` r"r)zMMapCache._fetchsy +E7!C59: =E ;))C DII #t~~- 4>>) KQ7 ?;? #IIII")D/7LM ,JAwC 1 w78GQZ$..0Fwr{T^^3dnnDdiiPD  & &$- 7 & LL"71:,a }BvhavQO  MM64. ) KK  w ' OOs7| +O% ,(::eC( (    LL+F84 5#D$6$6v$>? ,Q%c{  0$@A*+ 6$' , !' E  /xq?@*.,,vt*D 6$' Ezz%$$r$c@|jj}|d=|S)NrN)__dict__copyr!states r" __getstate__zMMapCache.__getstate__s  ""$ 'N r$cd|jj||j|_yrG)rrwrMrNrs r" __setstate__zMMapCache.__setstate__s" U#^^% r$)NNN)rr5rr6rr5rKz str | NonerJzset[int] | NonerLzMultiFetcher | Noner7r8)r7zmmap.mmap | bytearrayr'r:ryr:r7r;r7dict[str, Any]rrr7r8) r2r>r?r@rr#rMr)rr __classcell__r1s@r"rDrDasx8 D $"&-1 & & & &  &  &+ &  &1,<%| &r$rDc0eZdZdZdZdfd ZddZxZS)ReadAheadCachea!Cache which reads only when we get beyond a block of data This is a much simpler version of BytesCache, and does not attempt to fill holes in the cache or keep fragments alive. It is best suited to many small reads in a sequential order (e.g., reading lines from a file). readaheadcRt||||d|_d|_d|_y)Nr$r)rHr#rNr'ryr!rrrr1s r"r#zReadAheadCache.__init__s) GT2  r$c^|d}|||jkDr |j}||jk\s||k\ry||z }||jk\rM||jkr>|xjdz c_|j||jz ||jz S|j|cxkr|jkrOnnL|xj dz c_|j||jz d}|t |z}|j}n|xj dz c_d}t|j||jz}|xj||z z c_ |j|||_||_|jt |jz|_||jd|zSNrr$rP) rr'ryrrNrrxrurr r)r!r'rylparts r"r)zReadAheadCache._fetchse =E ;# /))C DII # %K DJJ 3$((? NNa N::edjj033CD D ZZ5 +488 + OOq O::edjj023D TNAHHE OOq OD$))S4>>12 ""cEk1"\\%-  ::DJJ/djj!n$$r$r4rr2r>r?r@rr#r)rrs@r"rrs D %r$rc0eZdZdZdZdfd ZddZxZS)FirstChunkCachezCaches the first block of a file only This may be useful for file types where the metadata is stored in the header, but is randomly accessed. firstcD||kDr|}t||||d|_yrG)rHr#rNrs r"r#zFirstChunkCache.__init__&s( t I GT2#' r$cn|xsd}||jkDrtjdyt||j}||jkr&|j |xj dz c_||jkDr@|xj|z c_|jd|}|d|j|_||dS|jd|j|_|xj|jz c_|j ||}||jkDrA|xj||jz z c_||j|j|z }|xjdz c_ |S|xj dz c_|xj||z z c_|j||S)Nrz,FirstChunkCache: requested start > file sizer$rP) rrnrorurrNrr rr)r!r'rydatars r"r)zFirstChunkCache._fetch-sh  499  LLG H#tyy! 4>> !zz!1$'..#5.<<3/D!%&6!7DJ<'!\\!T^^< **dnn<*::eC(DT^^#**cDNN.BB* T^^S99 NNa NK OOq O  & &#+ 5 &<<s+ +r$r4rrrs@r"rrs D(,r$rceZdZdZdZ d d fd ZdZd dZd dZddZ dfd Z dd Z xZ S) BlockCachea Cache holding memory as a set of blocks. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. blockcachect||||tj||z |_||_t j||j|_ yrG) rHr#mathceilr maxblocks functools lru_cache _fetch_block_fetch_block_cachedr!rrrrr1s r"r#zBlockCache.__init__dsR GT2yy !12 "#A9#6#6y#A$BSBS#T r$c6|jjSz The statistics on the block cache. Returns ------- NamedTuple Returned directly from the LRU Cache used internally. r cache_infor+s r"rzBlockCache.cache_infol''2244r$c$|j}|d=|S)Nrrrs r"rzBlockCache.__getstate__ws  ' ( r$c|jj|tj|d|j|_y)Nr)rrwrrrrrs r"rzBlockCache.__setstate__|s< U##J9#6#6u[7I#J   $  r$c|d}| |j}||jk\s||k\ry|j||||jz|dz |jzSr)r _read_cacher)r!r'rys r"r)zBlockCache._fetchsa =E ;))C DII # 3/#'dnn1L  r$c@||jkDrtd|d|jd||jz}||jz}|xj||z z c_|xjdz c_t j d|t|!||}|S)= Fetch the block of data for `block_number`. 'block_number=(' is greater than the number of blocks (rmrPzBlockCache fetching block %d) r ValueErrorrr rrninforHr))r! block_numberr'ryblock_contentsr1s r"rzBlockCache._fetch_blocks $,, & /))-a9  t~~-dnn$ ""cEk1" 1 2LAs3r$c ||jz}||jz}|dk(r |j}|xjdz c_||k(r|j|}|||S|j||dg}|jt |jt |dz||j |j|d|dj|S) Read from our block cache. Parameters ---------- start, end : int The start and end byte positions. start_block_number, end_block_number : int The start and end block numbers. rrPNr$rrrextendrtrprvjoin r!r'rystart_block_numberend_block_number start_posend_posblockouts r"rzBlockCache._read_cachesDNN* & a<nnG ! !1 1334FGE7+ +++,>? KLC JJ,,,q02BC  JJt//0@A(7K L88C= r$ rr5rr6rr5rr5r7r8rrr)rr5r7r; r'r5ryr5rr5rr5r7r;) r2r>r?r@rr#rrrr)rrrrs@r"rrKs, DMOUU'.U69UFIU U 5   $(!(!"(!8;(!OR(! (!r$rcZeZdZUdZdZded< d d fd Zd dZd dZxZ S) BytesCacheaKCache which holds data in a in-memory bytes object Implements read-ahead by the block size, for semi-random reads progressing through the file. Parameters ---------- trim: bool As we read more data, whether to discard the start of the buffer when we are more than a blocksize ahead of it. r;rrc`t||||d|_d|_d|_||_y)Nr$)rHr#rNr'rytrim)r!rrrrr1s r"r#zBytesCache.__init__s2 GT2 !% # r$c|d}| |j}||jk\s||k\ry|jc||jk\rT|jH||jkr9||jz }|xjdz c_|j|||z|z S|j r$t |j||j z}n|}||k(s||jkDry|j||jkrh|j||jkDrM|xj||z z c_|xjdz c_|j|||_||_n|jJ|jJ|xjdz c_||jkr|j|j|z |j kDr8|xj||z z c_|j|||_||_n4|xj|j|z z c_|j||j}||_||jz|_n|j||jkDr|j|jkDrn||jz |j kDr7|xj||z z c_|j|||_||_nR|xj||jz z c_|j|j|}|j|z|_|jt|jz|_||jz }|j|||z|z }|jrq|j|jz |j dzz}|dkDrC|xj|j |zz c_|j|j |zd|_|Sr) rr'ryrrNrrur rrrxr)r!r'ryoffsetbendnewrnums r"r)zBytesCache._fetchsJ =E ;))C DII # JJ "#$dhhTZZ'F NNa N::fv|e';< < >>tyy#"67DD 5=EDII- JJ %$**"4 HH dhh  & &$, 6 & OOq OeT2DJDJ::) ))88' '' OOq Otzz!88#txx#~'F..$,>.!%eT!:DJ!&DJ..$**u2DD.,,udjj9C!&DJ!$tzz!1DJ%$/88dii'488^dnn4..$,>.!%eT!:DJ!&DJ..$/A.,,txx6C!%c!1DJ::DJJ/#jj&3,"67 9988djj(dnnq.@ACQw dnns22 !ZZ(<(>?  r$c,t|jSrG)rxrNr+s r"__len__zBytesCache.__len__,s4::r$)T) rr5rr6rr5rboolr7r8r)r7r5) r2r>r?r@rrAr#r)rrrs@r"rrsT "D-!IM'.69AE GRr$rcXeZdZUdZdZded< d dfd Zd dZxZS) AllBytesz!Cache entire contents of the fileallrrct|||||P|xjdz c_|xj|jz c_|j d|j}||_y)NrPr)rHr#rr rrr)r!rrrrr1s r"r#zAllBytes.__init__5sY GT2 < OOq O  & &$)) 3 &<<499-D r$cJ|xjdz c_|j||S)NrP)rrr&s r"r)zAllBytes._fetchCs! !yyt$$r$)NNNN) rr:rzFetcher | Nonerr:rz bytes | Noner7r8r9) r2r>r?r@rrAr#r)rrs@r"rr0sX+D-!%"&!         %r$rceZdZUdZdZded< d d fd ZedZejdZedZ e jd Z d d Z xZ S)KnownPartsOfAFilea Cache holding known file parts. Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is data: dict A dictionary mapping explicit `(start, stop)` file-offset tuples with known bytes. strict: bool, default True Whether to fetch reads that go beyond a known byte-range boundary. If `False`, any read that ends outside a known part will be zero padded. Note that zero padding will not be used for reads that begin outside a known byte-range. partsrrc t||||||_|rt|j }|dg}|j |dg} |ddD]m\} } |d\} } | | k(r&| | f|d<| dxx|j | | fz cc<9|j | | f| j |j | | fott|| |_ yi|_ y)NrrPrl) rHr#strictsortedkeyspoprvdictzipr)r!rrrrrr old_offsetsoffsetsrJr'r(start0stop0r1s r"r#zKnownPartsOfAFile.__init__as GT2   -K"1~&Ghh{1~./F*12 ; t ' E>#)4.GBK2J$((E4="99JNNE4=1MM$((E4="9: ;S&12DIDIr$c:td|jDS)Nc32K|]}|d|dz yw)rPrNrB)rcrs r"rez)KnownPartsOfAFile.size..s211Q4!A$;2s)rqrr+s r"rzKnownPartsOfAFile.sizes2 222r$cyrGrBr!values r"rzKnownPartsOfAFile.size r$c,t|jSrG)rxrr+s r"rzKnownPartsOfAFile.nblockss499~r$cyrGrBrs r"rzKnownPartsOfAFile.nblocksrr$ctjd|||d}| |j}|xj||z z c_d}d}d}t |j D]T\}}||cxkr|krGnnD||cxkr|kr9nn6||z }|xj dz c_|j ||f|||z|z cS||krn|r7||kDr2|xjdz c_|jrt|d||z zz }||cxkr|kr:nn7|xj dz c_||z }|j ||f|||z|z }d}nu||kr/||kDr*|xj dz c_||j ||fz }nA||cxkr|kr6nn3|xj dz c_||j ||fd||z z}|cS|}W|xjdz c_|r|js |d||z zz}|St)NzKnown parts request %s %srr$FrPT) rnrorr rrrrrr) r!r'r(rstartedloc_oldloc0loc1offs r"r)zKnownPartsOfAFile._fetchs 0%> =E <99D ""dUl2" + JD$$$44+?4+?dl!#yy$.sS4Z%5GHHt|4'>1$;;$$w$.11u#t#!#dliit -cC$J4FG$+!#tyy$..%%!#DIItTl3MdTkBB G; < 1 4;;4'>22CJr$)NF) rr5rr6rr5rz#dict[tuple[int, int], bytes] | Nonerrrr r9) r2r>r?r@rrAr#propertyrsetterrr)rrs@r"rrHs,"D-!59   2   <33 [[   ^^  ,r$rcPeZdZdZGddeZd d dZd dZd dZddZ ddZ y ) UpdatableLRUzh Custom implementation of LRU cache that allows updating keys Used by BackgroundBlockCache c6eZdZUded<ded<ded<ded<y)UpdatableLRU.CacheInfor5hitsmissesmaxsizecurrsizeN)r2r>r?rArBr$r" CacheInfors    r$r ctj|_||_||_d|_d|_tj|_ yr) collectionsr_cache_func _max_size_hits_misses threadingLock_lock)r!funcmax_sizes r"r#zUpdatableLRU.__init__s<+6+B+B+D  !  ^^% r$c^|rtd|j|j5||jvrH|jj ||xj dz c_|j|cdddS ddd|j |i|}|j5||j|<|xjdz c_t|j|jkDr|jjdddd|S#1swYxYw#1swY|SxYw)Nz Got unexpected keyword argument rPFlast) TypeErrorrrr  move_to_endrr rrxrpopitem)r!argskwargsresults r"__call__zUpdatableLRU.__call__s >v{{}oNO O ZZ )t{{" ''- a {{4(  ) )" ) T,V, ZZ 0 &DKK  LLA L4;;$..0 ###/  0   ) ) 0  sA D)A#D"D"D,cb|j5||jvcdddS#1swYyxYwrG)rr )r!rs r" is_key_cachedzUpdatableLRU.is_key_cacheds* ZZ '4;;& ' ' 's%.c|j5||j|<t|j|jkDr|jj ddddy#1swYyxYw)NFr)rr rxrr)r!rrs r"add_keyzUpdatableLRU.add_keysV ZZ 0 &DKK 4;;$..0 ###/ 0 0 0s AA$$A-c|j5|j|jt|j|j |j cdddS#1swYyxYw)N)rrrr)rr rrxr rrr+s r"rzUpdatableLRU.cache_infosP ZZ >>T[[)ZZ|| "   s AAA&N))rzCallable[P, T]rr5r7r8)rzP.argsrzP.kwargsr7r)rr r7r)rrrr r7r8r7r) r2r>r?r@rr r#rr!r#rrBr$r"rrs, J &&'0 r$rceZdZUdZdZded< d d fd ZddZddZddZ dd Z ddfd Z dd Z xZ S)BackgroundBlockCachea Cache holding memory as a set of blocks with pre-loading of the next block in the background. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. If the next block is not in cache, it is loaded in a separate thread in non-blocking way. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. backgroundrrct||||tj||z |_||_t |j||_td|_ d|_ d|_ tj|_y)NrP max_workers)rHr#rrrrrrrr_thread_executor_fetch_future_block_number _fetch_futurerr_fetch_future_lockrs r"r#zBackgroundBlockCache.__init__ss GT2yy !12 "#/0A0A9#M 2q A6:'37"+.."2r$c6|jjSrrr+s r"rzBackgroundBlockCache.cache_info#rr$c<|j}|d=|d=|d=|d=|d=|S)Nrr-r.r/r0rrs r"rz!BackgroundBlockCache.__getstate__.s<  ' ( $ % . / / " & ' r$c|jj|t|j|d|_t d|_d|_d|_tj|_ y)NrrPr+) rrwrrrrr-r.r/rrr0rs r"rz!BackgroundBlockCache.__setstate__7sZ U##/0A0A5CU#V 2q A*.'!"+.."2r$c|d}| |j}||jk\s||k\ry||jz}||jz}d}d}|j5|j|jJ|jj rbt jd|jj|jj|jd|_d|_nKt||jcxkxr|knc}|r&|j}|j}d|_d|_ddd|?t jd|jj|j|t||dzD]}|j||dz} |j5|j]| |jkrN|jj| s3| |_|jj!|j"| d|_ddd|j%||||S#1swYxYw#1swY+xYw)Nrr$z3BlockCache joined background fetch without waiting.z(BlockCache waiting for background fetch.rPasync)rr)rrr0r/r.donernrrr#rrrprr!r-submitrr) r!r'ryrrfetch_future_block_number fetch_future must_joinrend_block_plus_1s r"r)zBackgroundBlockCache._fetch?s^ =E ;))C DII ##dnn4$..0$(!  $ $ 2!!-66BBB%%**,KK UV,,44**113T5T5T7;D3)-D&!%*::,+,!I !594S4S1'+'9'9 ;?7-1*7 2<  # KKB C  $ $ , ,##%'@  ""46F6JK 3L  $ $\ 2 3 ,a/  $ $ ""*$ 400>>?OP2B/%)%:%:%A%A%%'7&"   1-    o 2 2X  sC"H>7A*I >I IcB||jkDrtd|d|jd||jz}||jz}tj d|||xj ||z z c_|xj dz c_t|!||}|S)rrrrmz!BlockCache fetching block (%s) %drP) rrrrnrr rrHr))r!rlog_infor'ryrr1s r"rz!BackgroundBlockCache._fetch_blocks $,, & /))-a9  t~~-dnn$ 7<P ""cEk1" 1s3r$c ||jz}||jz}|xjdz c_||k(r|j|}|||S|j||dg}|jt |jt |dz||j |j|d|dj|S)rrPNr$rrs r"rz BackgroundBlockCache._read_cachesDNN* & ! !1 1,,-?@E7+ +++,>? KLC JJ,,,q02BC  JJt//0@A(7K L88C= r$rrr&rr<r)sync)rr5r=r=r7r;r)r2r>r?r@rrAr#rrrr)rrrrs@r"r(r(s2'D-&MO 3 3'. 369 3FI 3  3 53J X$(!(!"(!8;(!OR(! (!r$r(z!dict[str | None, type[BaseCache]]cachescr|j}|s |tvrtd|dt||t|<y)z'Register' cache implementation. Parameters ---------- clobber: bool, optional If set to True (default is False) - allow to overwrite existing entry. Raises ------ ValueError zCache with name z is already known: N)rr@r)clsclobberrs r"register_cacherDs= 88D tv~+D83Fvd|nUVVF4Lr$)F)rBztype[BaseCache]rCrr7r8)3 __future__rr rloggingrrTrrcollections.abcrconcurrent.futuresrr itertoolsroperatorr typingr r r r rrrEtyping_extensionsrrr getLoggerrnr5r;r6list MultiFetcherrrDrrrrrrrr(r@rArDcrBr$r"rQsy" #$9MM+#A A CL   + , C:u$ %c3h(%/0 < < ~K& K&\+%Y+%\+,i+,\}!}!@bbJ%y%0s sl971a4=9xK!9K!` )-) (    A1 r$