
    :hi                     T   d dl Z d dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlZd dlmZ d dlZd dlT d dlZ e	dg          d	             Z e	dg          d
             Zd Zd Z e	dg          d             Z e	dg          d             Z e	dg          d             Z e	dg          d             Z e	dg          d             Z e	dg          d             Z e	dg          d             Z e	dg          d             Z e	dg          d             ZdS )    N)settings)JsonResponse)require_http_methods)DevelopmentApplication)EXTRACT_STRUCTURED_DATA_PROMPT)*GETc                 <   	 t           j                            t          j        d          }t           j                            t          j        d          }t           j                            t          j        d          }t           j                            t          j        d          }t          j        |          }t          j        |          }|j        dd|j        j	        
                    dd           f         }|j        dd|j        j	        
                    dd           f         }|j                            |j                  }||         }||         }t          j        ||gd	
          }||                    d                   }	|                    d	
          }
|
                    |d           |	                    |d           t!          d|                     t          j        dz             |                     t          j        dz             t'          t)          |	                    t'          t)          |
                    d          S # t*          $ r-}t!          ddt          |           id          cY d}~S d}~ww xY w)z
    Merges two CSV files using only exactly matching column names.
    Removes duplicates and prevents creation of unwanted columns.
    z,Application Tables - 2025-07-20 08_21_55.csvzhistorical_scrape.csvmerged_output.csvzduplicate_entries.csvNz^UnnamedF)caseT)ignore_index)keepindexz7Merge completed successfully using exact column matches)messageoutput_fileduplicates_filenum_duplicatesnum_final_rowserrorMerge failed:   status)ospathjoinr   
MEDIA_ROOTpdread_csvloccolumnsstrcontainsintersectionconcat
duplicateddrop_duplicatesto_csvr   build_absolute_uri	MEDIA_URLintlen	Exception)requestfile1file2merged_filedup_filedf1df2common_columnscombined_dfduplicate_rows	unique_dfes               ,/home/esfera/Downloads/shao/shaoApp/views.pymergeCsvFilesViewr<      sj   'NX02`aaX02IJJgll8#68KLL7<< 35LMM k%  k%   gaaa#+/22:E2JJJJKgaaa#+/22:E2JJJJK 11#+>>.!.! ic
>>>$[%;%;%;%G%GH//T/BB	 	E222he444P"55h6HK^6^__&99(:LOf:fgg!#n"5"566!#i..11
 
   	  N N NW&?s1vv&?&?@MMMMMMMMMNs   I!I$ $
J."JJJc                    	 t           j                            t          j        d          }t           j                            t          j        d          }t           j                            t          j        d          }t          j        |          }t          j        |          }g d}||         }t          j        ||dd          }|                    |d	           |d
         	                                
                                }t          d|                     t          j        dz             t          |          d          S # t          $ r-}	t          ddt!          |	           id          cY d}	~	S d}	~	ww xY w)zE
    Enriches merged CSV data using determined application data.
    r   z%cumulative_applicationsdetermined.csvenriched_merged_output.csv)application_iddecisionlodgement_datedetermined_datecostcouncil_namer?   leftonhowFr   r@   z!Enrichment completed successfully)r   enriched_filenum_records_enrichedr   zEnrichment failed: r   r   N)r   r   r   r   r   r   r    merger)   notnasumr   r*   r+   r,   r.   r#   )
r/   merged_pathdetermined_pathenriched_output_path	merged_dfdetermined_dfcolsenriched_dfenriched_countr:   s
             r;   enrichMergedFileViewrV   A   s   
Sgll8#68KLL',,x':<cdd!w||H,?A]^^K,,	O44 kjj%d+hy-<LRXYYY/u===$Z06688<<>>:$778JMi8ijj$'$7$7
 
   	  S S SW&DCFF&D&DEcRRRRRRRRRSs   EE 
F"F;FFc                     	 t          j        | d          }t          j        |          st          |t           j                  rd S |                                S # t          $ r Y d S w xY w)Ncoerce)errors)r   to_datetimeisnull
isinstanceNaTdater.   )date_strr^   s     r;   
parse_dater`   a   ss    ~hx8889T?? 	jrv66 	4yy{{   tts   AA A 
A*)A*c                 >    t          j        |           s| dk    rd n| S )N )r   isna)vals    r;   safe_getre   j   s"    73<<53"9544#5    c                 P   	 t           j                            t          j        d          }t           j                            |          st          ddid          S t          j        |          }d\  }}}d}t          |          }t          d||          D ]}|j        |||z            }	t          d	||z  d
z    d||z   d
z
  |z              |	                                D ]K\  }
}	 t          |                    dd                    }|rt!          |          }ni }t"          j                            t          |                    d                    t          |                    d                    t          |                    d                    t)          |                    d                    t)          |                    d                    t          |                    d                    t          |                    d                    t          |                    d                    t          |                    d                    t          |                    d                    t          |                    d                    t          |                    d                    t          |                    d                    d          \  }}|r|d
z  }n|d
z  }# t*          $ r"}t          d|            |d
z  }Y d}~Ed}~ww xY w||z   |k     rt-          j        d           t          d |||||z   |z   d!          S # t*          $ r-}t          dd"t1          |           id#          cY d}~S d}~ww xY w)$zS
    Imports enriched CSV data into the database with improved error handling.
    enriched_merged_output2.csvr   z&enriched_merged_output2.csv not found.  r   )r   r   r      r   Processing batch    /descriptionrb   r?   application_web_urlr@   rA   rB   rC   council_name_xdevelopment_typenumber_of_dwellingsnumber_of_storeysnumber_of_placesnumber_of_unitsnumber_of_lots)application_urlr@   rA   rB   rC   rD   rq   rr   rs   rt   number_of_apartmentsnumber_of_subdivisions)r?   defaultszError processing row: N   z2Data import completed with improved error handling)r   createdupdatedfailedtotal_processedzData import failed: r   )r   r   r   r   r   existsr   r   r    r-   rangeilocprintiterrowsre   getextract_structured_datar   objectsupdate_or_creater`   r.   timesleepr#   )r/   	file_pathdfr|   r}   r~   
batch_size
total_rowsibatch_df_rowrn   
structuredobj
is_createdr:   s                    r;   importEnrichedDataViewr   m   s   
CTGLL!46STT	w~~i(( 	a*R S\_````[###* & 
WW
q*j11 ,	 ,	AwqZ/0Heama&7ee:
;RUV;VYc:ceefff"++-- $ $3#"*377="+E+E"F"FK # (%<[%I%I

%'
&<&D&U&U'/8H0I0I'J'J/7@U8V8V/W/W(01D1D(E(E.8AQ9R9R.S.S/9#''BS:T:T/U/U$,SWWV__$=$=,4SWW=M5N5N,O,O08HZ9[9[0\0\3;JNNK`<a<a3b3b19*..I\:]:]1^1^08HZ9[9[0\0\4<Z^^L]=^=^4_4_6>z~~N^?_?_6`6`" " 'V ' 'OC$ " %11    6166777aKFHHHH :~
* 
1K&069
 
   	  T T TW&ESVV&E&EFsSSSSSSSSSTsP   AM. BM. .HLM. 
L1L,&M. ,L11<M. .
N%8"N N% N%c           	      .   	 t           j                            t          j        d          }t           j                            t          j        d          }t           j                            t          j        d          }t           j                            |          rt           j                            |          st          ddid          S t          j        |          }t          j        |          }|	                    dd	id
           |
                    d	g          }||d	                             t                    j                                        dk             }t          j        ||g d         d	d          }|d                             t                                        d           |d<   |                    |d           t          d|                     t          j        dz             t'          t)          |                    t'          |d                                                                                   d          S # t.          $ r-}t          ddt          |           id          cY d}~S d}~ww xY w)z
    Merges enriched data with Name field from the scraper document based on Application ID.
    Filters out rows with missing Application ID before merging.
    Replaces Windows-style file path with /media/PDFs/ format.
    r>   z+Scraper Documents - 2025-07-20 08_21_54.csvenriched_with_name.csvr   z%One or both source files are missing.ri   r   zApplication IDr?   T)r"   inplace)subsetrb   )r?   NamePathrE   rF   r   c                     t          | t                    r]|                                 rIdd                    d |                     dd                              d          D                       z   ndS )Nz/media/rm   c                 l    g | ]1}|d |v |                                 rt          |          dk    /|2S ):   )isupperr-   ).0parts     r;   
<listcomp>zBmergeWithNameFromScraperView.<locals>.<lambda>.<locals>.<listcomp>   se     , , ,, "%, 26, EHIIQRN,, , ,rf   \rb   )r\   r#   stripr   replacesplit)ps    r;   <lambda>z.mergeWithNameFromScraperView.<locals>.<lambda>   s     Q$$	; *+	;i#(( , ,!"4!5!5!;!;C!@!@, , , # #   9;	 rf   Fr   z&Merge with Name completed successfullyr   )r   r   num_recordsnum_with_namer   r   N)r   r   r   r   r   r   r   r   r    renamedropnaastyper#   r   rK   applyr)   r*   r+   r,   r-   rL   rM   r.   )r/   enriched_pathscraper_docs_pathoutput_pathrT   
scraper_dfrQ   r:   s           r;   mergeWithNameFromScraperViewr      s   1NX%8:VWWGLL)<>kllgll8#68PQQ w~~m,, 	`BGNNCT4U4U 	`*Q R[^____ k-00[!233
 	#35E"FPTUUU "((1A0B(CC!+.>"?"F"Fs"K"K"O"U"U"W"W[]"]^ H999:	
 
 
	 &f-44S99??; ;
 
	& 	E222?"55h6HKc6cdds9~~.. 6!2!8!8!:!:!>!>!@!@AA	
 
   	  N N NW&?s1vv&?&?@MMMMMMMMMNs%   CI FI 
J'"J	JJc                 6   	 t           j                            t          j        d          }t           j                            |          st          ddid          S t          j        |          }d|j	        vs	d|j	        vrt          ddid	          S g }|
                                D ]\  }}|d         }t           j                            t          j        |                    d
                    }t          |           t           j                            |          r|                    |d                    t          dt          |          |d          S # t           $ r-}t          ddt#          |           id          cY d}~S d}~ww xY w)z
    Checks the existence of PDF files specified in the 'Path' column
    of enriched_with_name.csv and returns a list of application IDs
    for which the file exists.
    r   r   zMerged file not found.ri   r   r?   r   z(Required columns are missing in the CSV.  rm   zFile existence check complete.)r   num_files_foundapplication_idszFile check failed: r   N)r   r   r   r   r   r   r   r   r    r"   r   BASE_DIRlstripr   appendr-   r.   r#   )	r/   
input_pathr   existing_idsr   r   relative_path	full_pathr:   s	            r;   checkExistingPdfFilesViewr      s   SW\\("57OPP
 w~~j)) 	Q*B CCPPPP [$$2:- 	crz1I 	c*T U^abbbb kkmm 	; 	;FAsKMX%68L8LS8Q8QRRI)w~~i(( ;##C(8$9:::7"<00+
 
   	  S S SW&DCFF&D&DEcRRRRRRRRRSs+   AE! 8E! C	E! !
F+"FFFc                    	 t           j                            t          j        d          }t           j                            |          st          ddid          S t          j        |          }d|j	        vs	d|j	        vrt          ddid	          S g }d
}d
}g }|
                                D ]\  }}|d         }	|d         }
t          j        |
          s|
dk    r0t           j                            t          j        |
                    d                    }t           j                            |          r|                    |	|f           |                    |	d|
d           |dz  }d}t          |          }t!          d
||          D ]}||||z            }t#          d||z  dz    d||z   dz
  |z              |D ]P\  }	}	 t%          |          }|                    d          s]|                    |	|                    dd          |                    dd          |                    dd          |d           |dz  }t)          |d                   }|                    |	|||                    d          |                    d          |                    dd
          |                    dd
          d           |dz  }# t*          $ r6}|                    |	t-          |          |d           |dz  }Y d}~Jd}~ww xY w||z   |k     rt/          j        d           t           j                            t          j        d          }t3          |dd !          5 }t5          j        ||d"d#$           ddd           n# 1 swxY w Y   t          d%||||t          |          d&          S # t*          $ r-}t          dd't-          |           id(          cY d}~S d}~ww xY w))zX
    Extract data from PDF files with improved error handling and batch processing.
    r   r   !enriched_with_name.csv not found.ri   r   r?   r   CSV missing required columnsr   r   rb   rm   zPDF file not found)r?   r   r   rl      zProcessing PDF batch textpdf_typeunknown
confidencelowreasonzNo text extracted from PDF)r?   r   r   r   r   text_lengthpages_processed)r?   datar   r   r   r   r   Nextracted_applications.jsonwzutf-8)encodingr{   F)indentensure_asciiz5PDF extraction completed with improved error handling)r   r   
total_pdfssuccessfully_processedr~   total_recordszPDF extraction failed: r   )r   r   r   r   r   r   r   r   r    r"   r   rc   r   r   r   r-   r   r   extract_text_from_pdfr   !extract_structured_data_from_textr.   r#   r   r   openjsondump)r/   	input_csvr   result	processedr~   
valid_rowsr   r   app_idrel_pathpdf_pathr   r   r   batchextraction_resultextracted_datar:   r   fs                        r;   extractPdfDatar     s   
cWGLL!46NOO	w~~i(( 	\*M NWZ[[[[[##2:- 	Wrz1I 	W*H IRUVVVV	 
kkmm 	 	FAs)*F6{Hwx   HN w||H$5xs7K7KLLHw~~h'' !!68"45555&,1$    
 ! 
__
q*j11 ,	 ,	AqZ/0Ei!Z-!*;iizJ?VYZ?Z]g>giijjj$) $  $  # (=h(G(G%,0088 	!.4(9(=(=j)(T(T*;*?*?e*T*T&7&;&;HFb&c&c$,' '    !  &GGXY_G`%a%aNMM*0 . ($5$9$9*$E$E&7&;&;L&I&I'8'<'<]A'N'N+<+@+@ARTU+V+V# #    NII       MM*0!$Q (# #   
 aKFFFFFF  :~
* 
1 gll8#68UVV+sW555 	?Ifa>>>>	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? N&$&/ [[
 
   	  W W WW&HA&H&HIRUVVVVVVVVVWs   AO 8O D=O B KO BKO 
L'+LO LAO ;N O  N$$O 'N$(%O 
P"P :P Pc           
      &   	 d}t          d           t          |          }t          d||t          |o|                    d                     d          S # t
          $ r-}t          ddt          |           id          cY d	}~S d	}~ww xY w)
zI
    Test view to validate AI extraction functions with sample data.
    z9Construct 3 double storey dwellings with 6 bedrooms totalz)Testing AI extraction with sample data...zAI extraction test completedr   )r   
test_inputr   successTest failed: r   r   N)r   r   r   boolr   r.   r#   )r/   test_descriptionr   r:   s       r;   testAiExtractionViewr     s    
MV9:::()9::5*$F>6::g+>+>'>??	
 
   	  M M MW&>c!ff&>&>?LLLLLLLLLMs   AA 
B#"BBBc           
         	 i }g d}|D ]}t           j                            t          j        |          }t           j                            |          t           j                            |          rt           j                            |          ndd||<   t          j        	                                }t          j        
                    d          
                    d          	                                }t          d|||t          |dk    r||z  d	z  ndd
          dd          S # t          $ r-}t          ddt          |           id          cY d}~S d}~ww xY w)z?
    Get the current status of data processing operations.
    )rh   r   r   r   )r   sizeT)development_type__isnullrb   )rq   zProcessing status retrievedd   r{   )total_applications applications_with_extracted_datacompletion_percentage)r   files_statusdatabase_statsr   zStatus check failed: r   r   N)r   r   r   r   r   r   getsizer   r   countexcluder   roundr.   r#   )r/   r   required_filesfilenamer   r   applications_with_datar:   s           r;   getProcessingStatusViewr     s   
!U
 
 
 ' 	 	HX%8(CCI'..3368gnnY6O6OV	222UV& &L"" 4;AACC!7!?!G!G%) "H "
 "

'2'
&
&uuww 	 4(&84J).fx{|f|  0D0FI[0[^a0a0a  CD  FG  *H  *H 
 
   	  U U UW&Fc!ff&F&FGPSTTTTTTTTTUs   D.D1 1
E(;"E#E(#E(c                    	 t           j                            t          j        d          }t           j                            |          st          ddid          S t          j        |          }d|j	        vs	d|j	        vrt          ddid	          S d
d
d
d
d
d
dd
d
d
dd
d
d
ddd
idd
idg g g dd}d}t          |          }t          d
||          D ]}|j        |||z            }t          d||z  dz    d||z   dz
  |z              |                                D ]0\  }}	|	d         }
|	d         }|dxx         dz  cc<   t          j        |          s|dk    r|dxx         dz  cc<   Qt           j                            t          j        |                    d                    }t           j                            |          s|dxx         dz  cc<   |dxx         dz  cc<   	 t%          |          }|                    dd          }|                    dd          }||d         v ra|d         |         dxx         dz  cc<   |dk    r|d         |         dxx         dz  cc<   n"|dk    r|d         |         d xx         dz  cc<   |dv rnt          |d!         |                   d"k     rO|d!         |                             |
|||                    d#d
          |                    d$d
          d%           # t*          $ r<}|d         d         dxx         dz  cc<   t          d&| d'|            Y d(}~*d(}~ww xY w||z   |k     rt-          j        d           |d         }|d
k    rD|d         D ];}|d         |         d         }t1          ||z  d)z  d*          |d         |         d+<   <t          d,||d
k    r|d         d-         d+         nd
|d
k    r|d         d.         d+         nd
|d
k    r|d         d/         d+         nd
d0d1          S # t*          $ r-}t          dd2t3          |           id3          cY d(}~S d(}~ww xY w)4z]
    Analyze PDF types across the dataset to understand digital vs scanned distribution.
    r   r   r   ri   r   r?   r   r   r   r   )r   high_confidencemedium_confidencer   digitalscannedmixedr   r   )r  r  r  )total_filesfiles_foundfiles_not_found	pdf_typessample_files
   zAnalyzing PDF batch rl   rm   r  rb   r	  r  typer   r   r   r
  highr  mediumr  r  rj   text_densityavg_text_per_page)r?   r   r   r  r  zError analyzing PDF : Nr   r{   
percentagezPDF type analysis completedr  r  r  )digital_percentagescanned_percentagemixed_percentager   analysissummaryzPDF analysis failed: r   )r   r   r   r   r   r   r   r   r    r"   r-   r   r   r   r   rc   r   r   detect_pdf_typer   r   r.   r   r   r   r#   )r/   r   r   analysis_resultsr   r   r   r   r   r   r   r   r   pdf_infor   r   r:   total_analyzedr   s                      r;   analyzePdfTypesViewr    s   
fUGLL!46NOO	w~~i(( 	\*M NWZ[[[[[##2:- 	Wrz1I 	W*H IRUVVVV  %&1STUU%&1STUU#$QRSS!1#Q<   
 
& 
WW
q*j11 2	 2	AwqZ/0HhJ):hhj:>UXY>Y\f=fhhiii"++-- *B *B3-.v; ///14///78$$ B $%67771<7777<<(98??3;O;OPPw~~h// $%67771<777 ///14///B.x88H'||FI>>H!)lE!B!BJ  #3K#@@ ^(5h?HHHAMHHH%/ ^,[9(CDUVVVZ[[VVVV'83 ^,[9(CDWXXX\]]XXX  #BB sK[\jKkltKuGvGvyzGz (8BII.4$,*4,4LL,K,K19>QST1U1UK K    ! B B B$[1':7CCCqHCCC@@@Q@@AAAAAAAAB
 :~
* 
1 *-8A 	q,[9 q q(5h?HHMuWeOeilNlnoHpHp -h7EE4(`nqr`r&y&6{&CI&N|&\&\xy`nqr`r&y&6{&CI&N|&\&\xy\jmn\n$u$4[$A'$J<$X$Xtu 
 
   	  U U UW&Fc!ff&F&FGPSTTTTTTTTTUsW   AP0 8P0 EP0 7DLP0 
M1MP0 MCP0 0
Q':"Q"Q'"Q'c                    	 t           j                            t          j        dddd          }t           j                            |          st          ddid          S t          |          }t          |          }t          d	d||	                    d
          |	                    d          |	                    dd          t          |	                    d                    |	                    d          r!|	                    dd          dd         dz   nddd          S # t          $ r-}t          ddt          |           id          cY d}~S d}~ww xY w)zJ
    Test view to validate PDF type detection with a sample PDF file.
    COUNCIL DATACity of Darebinz
D-101-2023z*1 - Advertised Document - Title Search.pdfr   z Sample PDF not found for testingri   r   z!PDF type detection test completedr   r   r   r   r   rb   N   z...)r   r   r   has_textsample_text)r   sample_filepdf_type_detectiontext_extractionr   r   )r   r   r   r   r   r   r   r  r   r   r   r.   r#   )r/   sample_pdf_pathr  r   r:   s        r;   testPdfTypeDetectionViewr)  6  s   
M',,x':NL]_k  nZ  [  [w~~o.. 	[*L MVYZZZZ #?33 2/BB:G"*-11*==/33LAA044]AFF !2!6!6v!>!>??RcRgRghnRoRow044VR@@#FNNuw   	
 
   	  M M MW&>c!ff&>&>?LLLLLLLLLMs%   AD) !CD) )
E 3"EE E c                 
   	 ddl }ddl}|j                            t          j        dd          }|j                            |          st          ddid          S |j                            |d	d
          }|                     |d          }|st          ddid          S t          |          dddddg ddddg ddddg ddg ddg ddi dddddddddddddd}d}t          |          }t          d||          D ]!}	||	|	|z            }
t          d|	|z  dz    d||z   dz
  |z              |
D ]}	 |j                            |          }|j                            |t          j                  }|dk     r|d         d         dxx         dz  cc<   nb|d k     r|d         d!         dxx         dz  cc<   n?|d"k     r|d         d#         dxx         dz  cc<   n|d         d$         dxx         dz  cc<   |                    d          }d}|D ]}|                    d%          rd&|v r|} n t          |          }|                    d'd(          }|                    d)d*          }||d+         v r|d+         |         dxx         dz  cc<   |d,k    r|d+         |         d-xx         dz  cc<   n"|d!k    r|d+         |         d.xx         dz  cc<   t          |d+         |         d/                   d0k     r||d+         |         d/                             |t%          |d1z  d          |||                    d2d          |                    d3d          |                    d4d          d5           |r||d6         vrdddddd7|d6         |<   |d6         |         d8xx         dz  cc<   |d6         |         d9xx         |d1z  z  cc<   |d:k    r|d6         |         d;xx         dz  cc<   nE|d<k    r|d6         |         d=xx         dz  cc<   n"|d>k    r|d6         |         d?xx         dz  cc<   |d@xx         dz  cc<   # t&          $ r}t          dA| dB|            |dCxx         dz  cc<   |d+         d         dxx         dz  cc<   |d+         d         d/                             |j                            |t          j                  t)          |          dD           Y d}~d}~ww xY w|	|z   |k     rt+          j        d           #|d@         }|dk    rD|d+         D ];}|d+         |         d         }t%          ||z  dEz  dF          |d+         |         dG<   <t/          |d6                                         dH dI          }t3          |ddJ                   |d6<   t          dK|||dk    r|d+         d:         dG         nd|dk    r|d+         d<         dG         nd|dk    r|d+         d>         dG         ndt%          |t          |          z  dEz  dF          dLdM          S # t&          $ r-}t          ddNt)          |           idO          cY d}~S d}~ww xY w)PzK
    Analyze all PDF files in the City of Darebin directory structure.
    r   Nr   r!  r   z#City of Darebin directory not foundri   r   z**z*.pdfT)	recursivez/No PDF files found in City of Darebin directory)r   r  r  files)r   r,  r  z0-100KB)r   
size_rangez	100KB-1MBz1MB-10MBz10MB+)smallr  large
very_large)r  files_processedfiles_failedr
  applications
file_sizesrj   rk   rl   rm   i  r4  r.  r   i   r  i   r/  r0  zD--r  r   r   r   r
  r  r  r  r,  r  i   r  r  total_pages)r   size_kbr   r   r  r  r6  r3  )r  digital_filesscanned_filesmixed_filestotal_size_kbr  r;  r  r8  r  r9  r  r:  r1  zError processing r  r2  )r   r   r   r{   r  c                     | d         d         S )Nrl   r   )xs    r;   r   z(analyzeDarebinPdfsView.<locals>.<lambda>  s    !A$}- rf   )keyreverse   z&City of Darebin PDF analysis completed)r   r  r  r  success_rater  zDarebin PDF analysis failed: r   )globos.pathr   r   r   r   r   r   r-   r   r   r   relpathr   
startswithr  r   r   r   r.   r#   r   r   sorteditemsdict)r/   rC  r   darebin_basepdf_pattern	pdf_filesr  r   r  r   batch_filesr   	file_sizer   
path_partsr   r   r  r   r   r:   r   r   sorted_appss                           r;   analyzeDarebinPdfsViewrQ  Y  s	   
^] w||H$7IZ[[w~~l++ 	^*O PY\]]]] gll<w??IIkTI::	 	j*[ \ehiiii y>> %&1ST_abb%&1ST_abb#$QR]_``#$r22%&44  #$I>>$%[AA#$J??()AA	 
 
* 
)nnq+z22 W	 W	A#Aa*n$45Kfama&7ff;;SVW;WZd:dffggg' O ON " 9 9I$&GOOHh>Q$R$RM !:- S(6w?HHHAMHHHH"[0 S(6x@IIIQNIIII"%55 S(6w?HHHAMHHHH(6|DWMMMQRRMMM "/!4!4S!9!9J!F * " "??400 "SD[ "%)F!E  /x88H'||FI>>H!)lE!B!BJ  #3K#@@ (5h?HHHAMHHH%/ ^,[9(CDUVVVZ[[VVVV'83 ^,[9(CDWXXX\]]XXX /<XFwOPPSUU 	,[9(CGLSS(5+0T1A1+E+E*0.808^Q0O0O5=\\BUWX5Y5Y/7||M1/M/MU U     Y!)9.)II /01212/012H H,^<VD )8@OOOSTTOOO(8@QQQU^aeUeeQQQ#y0 Y,^<VD_UUUYZZUUUU%2 Y,^<VD_UUUYZZUUUU%0 Y,^<VD]SSSWXXSSS$%67771<7777    =h==!==>>>$^4449444$[1':7CCCqHCCC$[1':7CJJ "(:M N N!$QL L        	 :~+ 
1 ++<=Q 	r,[9 r r(5h?HHMuWfOfjmNmopHqHq -h7EE ^,2244--
 
 

 ,0CRC0@+A+A(?(-`ors`s&z&6{&CI&N|&\&\yz`ors`s&z&6{&CI&N|&\&\yz\kno\o$v$4[$A'$J<$X$Xuv %Y'G3&NPQ R R 

 

 
 
 
	  ] ] ]W&Nc!ff&N&NOX[\\\\\\\\\]sY   AY AY %BY LQY 
T
$BT?Y T

E Y 
Z"Y=7Z=Z)r   r   pandasr   django.confr   django.httpr   django.views.decorators.httpr   shaoApp.modelsr   openaishaoApp.promptsr   reshaoApp.functionsr   r<   rV   r`   re   r   r   r   r   r   r   r  r)  rQ  r=  rf   r;   <module>r[     s   				                  $ $ $ $ $ $ = = = = = = 1 1 1 1 1 1  : : : : : : 				     
 ug,N ,N ,N^ ugS S S>  6 6 6 ugGT GT GTV ug7N 7N 7Nt ug%S %S %SP uggW gW gWT ugM M M, ug%U %U %UP ugjU jU jUZ ugM M MD ugb] b] b] b] b]rf   