tjyKdZddlmZddlZddlmZmZddlmZddl m Z ddl m Z Gdd e Z Gd d ZGd d eZGddeZGddZdS)zMarkdown text splitters.) annotationsN)Any TypedDictDocument)Language)RecursiveCharacterTextSplitterc$eZdZdZdfd ZxZS)MarkdownTextSplitterz=Attempts to split the text along Markdown-formatted headings.kwargsrreturnNonec ||tj}tjdd|i|dS)z$Initialize a `MarkdownTextSplitter`. separatorsN)get_separators_for_languagerMARKDOWNsuper__init__)selfr r __class__s _/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/langchain_text_splitters/markdown.pyrzMarkdownTextSplitter.__init__sA55h6GHH 99J9&99999)r rr r)__name__ __module__ __qualname____doc__r __classcell__)rs@rr r sCGG::::::::::rr c:eZdZdZ dddZddZddZddZdS)MarkdownHeaderTextSplitterz4Splitting markdown files based on specified headers.FTNheaders_to_split_onlist[tuple[str, str]]return_each_linebool strip_headerscustom_header_patternsdict[str, int] | Noner rcd||_t|dd|_||_|pi|_dS)aCreate a new `MarkdownHeaderTextSplitter`. Args: headers_to_split_on: Headers we want to track return_each_line: Return each line w/ associated headers strip_headers: Strip split headers from the content of the chunk custom_header_patterns: Optional dict mapping header patterns to their levels. For example: `{"**": 1, "***": 2}` to treat `**Header**` as level 1 and `***Header***` as level 2 headers. c,t|dS)Nr)len)splits rz5MarkdownHeaderTextSplitter.__init__..2s3uQx==rT)keyreverseN)r#sortedr!r%r&)rr!r#r%r&s rrz#MarkdownHeaderTextSplitter.__init__sM(!1$* %@%@$$ $ $  +&<&B###rlinestrsepc F|jvrdStj}d|d|d|d|d }tj||}|rZ|d}|r1t fd|d d Dsd SdS) zCheck if line matches a custom header pattern. Args: line: The line to check sep: The separator pattern to match Returns: `True` if the line matches the custom pattern format F^z(?!z )(.+?)(?z?MarkdownHeaderTextSplitter._is_custom_header..Ts'"N"N18"N"N"N"N"N"Nr T)r&reescapematchgroupstripallreplace)rr0r2 escaped_seppatternrAcontents ` r_is_custom_headerz,MarkdownHeaderTextSplitter._is_custom_header9s d1 1 15inn  T S S S S{ S S[ S S S $''  kk!nn**,,G s"N"N"N"NW__S"5M5M"N"N"NNN turlineslist[LineType]list[Document]c4g}|D]}|r8|dd|dkr |ddxxd|dzz cc<=|r|dd|dkrt|ddt|dkri|ddddddkr8|js1|ddxxd|dzz cc<|d|dd<|| d|DS) zCombine lines with common metadata into chunks. Args: lines: Line of text / associated header metadata Returns: List of `Document` objects with common metadata aggregated. metadatarHz  r#cHg|]}t|d|d SrHrO page_contentrOrr:chunks r zHMarkdownHeaderTextSplitter.aggregate_lines_to_chunks..?    % "2U:=N O O O   r)r*r+r%append)rrJaggregated_chunksr0s raggregate_lines_to_chunksz4MarkdownHeaderTextSplitter.aggregate_lines_to_chunksXsn-/ / /D! /%b)*5j9III ""%i000FT)_4LL0000! /%b)*5j9III)"-j9::SjAQ=R=RRR%b))4::4@@DQG3NN*O""%i000FT)_4LL000484D!"%j11"((....  *    rtextc|d}g}g}i}g}i}d}d} |D]E} | } dttj| } |sM| dr| ddkrd}d} n3| drd}d} n| | rd}d} |r|| |j D]\} } | | o8t| t| kp| t| dk}| | | }|s|r| | |j vr|j | }n| d }|r_|d d |krM| }|d |vr| |d |r|d d |kM|r8| t| t|  }n)| t| d }|| |d}|||d|| <|rQ|d||d||js|| nm| r|| nS|rQ|d||d||}G|r+|d||d|js||Sd|DS)zSplit markdown file. Args: text: Markdown file Returns: List of `Document` objects. rPFr>z```r7Tz~~~r=NrQrNlevelname)r_r`datara)rHrOcHg|]}t|d|d SrSrrVs rrXz9MarkdownHeaderTextSplitter.split_text..rYr)r+rCjoinfilterr1 isprintable startswithcountrZr!r*rIr&popcopyclearr%r#r\)rr]rJlines_with_metadatacurrent_contentcurrent_metadata header_stackinitial_metadata in_code_block opening_fencer0 stripped_liner2r`is_standard_headeris_custom_headercurrent_header_level popped_header header_textheaders r split_textz%MarkdownHeaderTextSplitter.split_texts 4  /1&(+-*, +-  d 7d 7D JJLLMGGF3?M$J$JKKM # ++E22*}7J7J57Q7QUV7V7V$(M$)MM"--e44*$(M$)M))-88 # % "  &&}555"5L ,L , T%2%=%=c%B%B& &&#c((2TmCHH6MQT6T# $(#9#9-#M#M &8)98'$"===373Ns3S003699S>>0) L ,R 0 9=Q Q Q-9,<,<,>,>M -V48HHH 0 4 4]65J K K K) L ,R 0 9=Q Q Q,L+8CCHH98L*M*S*S*U*UKK+8C *C*I*I*K*KK&:$($/.. %++F33317(.'0+22+/99_+E+E,<,A,A,C,C (--///->'..}===Eq8t! ,#**=9999$,'..'+yy'A'A(8(=(=(?(? $))+++/4466      & &#yy99 0   $ G112EFF F  ,    r)FTN) r!r"r#r$r%r$r&r'r r)r0r1r2r1r r$)rJrKr rLr]r1r rL)rrrrrrIr\ryrrrr r s>> "'"8< CCCCC>>, , , , \R R R R R R rr c(eZdZUdZded<ded<dS)LineTypezLine type as `TypedDict`.zdict[str, str]rOr1rHNrrrr__annotations__rrrr|r|s+##LLLLLrr|c2eZdZUdZded<ded<ded<dS) HeaderTypezHeader type as `TypedDict`.intr_r1r`raNr}rrrrr"s1%%JJJ III IIIIIrrczeZdZdZ dd d Zd!dZd"dZd#dZd$dZd%dZ e d%dZ e d%dZ dS)&&ExperimentalMarkdownSyntaxTextSplitteraAn experimental text splitter for handling Markdown syntax. This splitter aims to retain the exact whitespace of the original text while extracting structured metadata, such as headers. It is a re-implementation of the `MarkdownHeaderTextSplitter` with notable changes to the approach and additional features. Key Features: * Retains the original whitespace and formatting of the Markdown text. * Extracts headers, code blocks, and horizontal rules as metadata. * Splits out code blocks and includes the language in the "Code" metadata key. * Splits text on horizontal rules (`---`) as well. * Defaults to sensible splitting behavior, which can be overridden using the `headers_to_split_on` parameter. Example: ```python headers_to_split_on = [ ("#", "Header 1"), ("##", "Header 2"), ] splitter = ExperimentalMarkdownSyntaxTextSplitter( headers_to_split_on=headers_to_split_on ) chunks = splitter.split(text) for chunk in chunks: print(chunk) ``` This class is currently experimental and subject to change based on feedback and further development. NFTr!list[tuple[str, str]] | Noner#r$r%r rcg|_td|_g|_||_|rt ||_nddddddd |_||_d S) aInitialize the text splitter with header splitting and formatting options. This constructor sets up the required configuration for splitting text into chunks based on specified headers and formatting preferences. Args: headers_to_split_on: A list of tuples, where each tuple contains a header tag (e.g., "h1") and its corresponding metadata key. If `None`, default headers are used. return_each_line: Whether to return each line as an individual chunk. Defaults to `False`, which aggregates lines into larger chunks. strip_headers: Whether to exclude headers from the resulting chunks. r>rUzHeader 1zHeader 2zHeader 3zHeader 4zHeader 5zHeader 6)rQz##z###z####z#####z######N)chunksr current_chunkcurrent_header_stackr%dictsplittable_headersr#)rr!r#r%s rrz/ExperimentalMarkdownSyntaxTextSplitter.__init__Ms}*') %2666;=!*  &*+>&?&?D # #  !"#$ ''D #!1rr]r1rLc|jtd|_|j|d}|rp|d}||}||}| |}|r~| |j s|jxj |z c_ t|d}|d}|||n|rk| ||||j_ |d|jjd<| n,|r| n|jxj |z c_ |p| |jrd |jDS|jS) a[Split the input text into structured chunks. This method processes the input text line by line, identifying and handling specific patterns such as headers, code blocks, and horizontal rules to split it into structured chunks based on headers, code blocks, and horizontal rules. Args: text: The input text to be split into chunks. Returns: A list of `Document` objects representing the structured chunks of the input text. If `return_each_line` is enabled, each line is returned as a separate `Document`. r>rT)keependsrr7Codecg|]J}|jD].}||t||j/KS)rT)rU splitlinesisspacerrO)r:rWr0s rrXzEExperimentalMarkdownSyntaxTextSplitter.split_text..su!.99;; !%  dU^DDDr)rrjrrrrrh _match_header _match_code _match_horz_complete_chunk_docr%rUr*rB_resolve_header_stack_resolve_code_chunkrOr#) rr] raw_linesraw_line header_match code_match horz_match header_depthrws rryz1ExperimentalMarkdownSyntaxTextSplitter.split_textts' %2666 !'')))OOTO22  < }}Q''H--h77L))(33J))(33J <((***)@&33x?33 #<#5#5a#8#899 *0033 **<EEEE <((***262J2Ji33"/7A6F6Fq6I6I"+F3((**** <((****"//8;//3 <6   """   ![  {rrrrwct|jD]$\}\}}||kr|jd||_n%|j||fdSr9) enumeraterrZ)rrrwidepth_s rrz)rhr)rrrrWrs rrz:ExperimentalMarkdownSyntaxTextSplitter._resolve_code_chunksV  }}Q''H X E))    rrc$|jj}|rl|sX|jD]1\}}|jd|z}||jj|<2|j|jtd|_dS)NrQr>r) rrUrrrgetrOrrZr)r chunk_contentrvalue header_keys rrz:ExperimentalMarkdownSyntaxTextSplitter._complete_chunk_docs*7  3!6!6!8!8 3 $ 9 @ @ u!488uEE :?"+J77 K  t1 2 2 2%2666rr0re.Match[str] | Nonecptjd|}|r|d|jvr|SdS)Nz^(#{1,6}) (.*)r7)r?rArBr)rr0rAs rrz4ExperimentalMarkdownSyntaxTextSplitter._match_headers=*D11  U[[^^t'>>>LtrcTfddD}td|DdS)Nc:g|]}tj|Srr?rAr:ruler0s rrXzFExperimentalMarkdownSyntaxTextSplitter._match_code..s%OOOD28D$''OOOr)z^```(.*)z^~~~(.*)c3K|]}||V dSr9rr:rAs rr<zEExperimentalMarkdownSyntaxTextSplitter._match_code..'99u59U999999rnextr0matchess` rrz2ExperimentalMarkdownSyntaxTextSplitter._match_codes<OOOO4NOOO999994@@@rcTfddD}td|DdS)Nc:g|]}tj|Srrrs rrXzFExperimentalMarkdownSyntaxTextSplitter._match_horz..s2   %)BHT4    r)z ^\*\*\*+\nz^---+\nz^___+\nc3K|]}||V dSr9rrs rr<zEExperimentalMarkdownSyntaxTextSplitter._match_horz..rrrrs` rrz2ExperimentalMarkdownSyntaxTextSplitter._match_horzsL    -T   999994@@@r)NFT)r!rr#r$r%r$r rrz)rrrwr1r r)rr1rrr r1)r r)r0r1r r) rrrrrryrrrr staticmethodrrrrrrr*s  H=A!&" %1%1%1%1%1N<<<<|FFFF 7 7 7 7AAA\AAAA\AAArr)r __future__rr?typingrrlangchain_core.documentsrlangchain_text_splitters.baser"langchain_text_splitters.characterr r r r|rrrrrrsj"""""" !!!!!!!!------222222MMMMMM:::::9:::A A A A A A A A HywAwAwAwAwAwAwAwAwAwAr