#h:^ddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z ddlZddlmZmZmZddlZddlZddlmZmZmZmZejejZd Zd Zd ed efd Zd de de!de"fdZ#de d efdZ$de d efdZ%de d efdZ&de d e fdZ'de d efdZ(dZ)d!dZ*dS)"N)settings) JsonResponse)require_http_methods)DevelopmentApplication)EXTRACT_STRUCTURED_DATA_PROMPTTITLE_SEARCH_EXTRACTION_PROMPT"APPLICATION_FORM_EXTRACTION_PROMPT)RateLimitErrorAPIErrorAPIConnectionErrorAPITimeoutError)api_keycr tj|S#t$rYdSwxYw)N)pd to_datetimedate Exception)date_strs 0/home/esfera/Downloads/shao/shaoApp/functions.py parse_datersE~h'',,... tts %( 66ctj|s|dks|dSt|S)z7Safely get value, handling None, NaN, and empty stringsN)risnastrstrip)vals rsafe_getrs? ws||sbyCt s88>>  datareturnct|tsiSddddddd}i}|D]\}}|||}|drX ||dkrd||<n,t t t|||<l#ttf$rd||<YwxYw||dkrd||<t| ||<|S)z7Validate and clean structured data from OpenAI responserr)development_typenumber_of_dwellingsnumber_of_storeysnumber_of_placesnumber_of_unitsnumber_of_lots number_of_) isinstancedictitemsget startswithintfloatr ValueError TypeErrorr)rexpected_fieldsvalidated_datafield default_valuevalues rvalidate_structured_datar7"sJ dD ! !   ON / 5 5 7 7;;} ..   L ) ) ; *CERKC,-N5)),/c%jj0A0A,B,BN5) * * * *()u%%% * ;  ;(*u%%(+E (8(8(:(:u%% s(:B##B<;B<?prompt max_retries base_delayc xt|D]} tjjdd|dgdd}|jdjj}|dfcS#t$r}||d z krT|d |zztj dd z}td |d d |d zd|dtj|Yd}~dd|dt|fcYd}~cSd}~wt t"t$f$r}||d z krU|d |zztj dd z}td|d d |d zd|dtj|Yd}~\dd|dt|fcYd}~cSd}~wt&$r }ddt|fcYd}~cSd}~wwxYwdS)z@Make OpenAI API request with exponential backoff and retry logiczgpt-4o-2024-05-13user)rolecontentg?i)modelmessages temperature max_tokensrNzRate limit hit, retrying in z.2fz seconds... (attempt /)zRate limit exceeded after z attempts: zAPI error, retrying in zAPI error after zUnexpected error: )NzMax retries exceeded)rangeclientchat completionscreatechoicesmessager@rr randomuniformprinttimesleeprr r r r)r:r;r<attemptresponser@edelays rmake_openai_request_with_retryrYIs%% 7 7 7{.55)#)f==> 6H&q)19??AAGD=  [ [ [q( ["a7l3fnQ6J6JJqUqqqU\_`U`qqcnqqqrrr 5!!!Z+ZZRUVWRXRXZZZZZZZZZZZ,o> Q Q Qq( Q"a7l3fnQ6J6JJllllPWZ[P[ll^illlmmm 5!!!P PPAPPPPPPPPPPP 7 7 76c!ff666 6 6 6 6 6 6 6 6 7 ( 'sPAA// F79AC4C4,F74F7AF ,F F7 F7F2*F72F7 descriptionc|r|siSd}t||kr |d|dz}tj|}t |\}}|rt d|iS|st diS |dr+tjdd |}| d}tj |}t|}|S#tj $r0}t d |t d |icYd}~Sd}~wt$r}t d |icYd}~Sd}~wwxYw) zCExtract structured data from description with proper error handlingiN...rZOpenAI request failed: Empty response from OpenAI``` ^```[a-z]*\n?rzJSON decoding error: Raw content: z%Unexpected error in data processing: )rlenrformatrYrRr-resubrstripjsonloadsr7JSONDecodeErrorr)rZ max_lengthr:r@errorrr3rWs rextract_structured_datarmqs k//11 J ;*$7!+:+.6 + 2{ K K KF4F;;NGU  ///000  *+++    e $ $ ,f-r7;;GnnU++Gz'""2$77   )a))*** 'g''(((  9a99::: s1 A$C11E%D+%E+ E8E EEpdf_pathc  tj|5}t|j}d}d}|jD]T}|}|rDetect if PDF is digital (text-based) or scanned (image-based)r rEgffffff?ddigitalhighg333333?2mediummixedlowscannedr8)type confidence total_pagespages_with_text text_densityavg_text_per_pagetotal_text_lengthNunknown)ryrzrlr{r|r}r~r) pdfplumberopenrcpages extract_textrroundrr) rnpdfr{r|rpage page_textr}r~pdf_typerzrWs rdetect_pdf_typersH2 _X & &% #ci..KO !   8 8 --// 8Y__%6%6!7!7"!<8#q(O%Y7%=H!OR?[88QRLCNQR? Y 1K ? ?XY s" $'83'> $$# $ $):R)? $$% ! $&7"&< $"" $# !(*#2 %lA 6 6%*+PDF appears to be scanned/image-based with no extractable text)textrrzreason Nr|)rrrz text_lengthpages_processedz5No text could be extracted despite PDF type detectionrlrw)rrrzrl) rrrrrrrcrr)rnpdf_inforrrrextracted_textrWs rextract_text_from_pdfrs. "8,, F y ( Xl-Cv-M %$Z   _X & & -#  - - --// -I,,D - - - - - - - - - - - - - - - -   &$V,&|4">22#+,=#>  %$Q      VV          sL-C C )A<0 C <BC B?C C C3C.(C3.C3rc|gd}gd}tfd|D}tfd|D}|dkrdS|dkrdS||krdS||krdSdS) z.Detect document type based on content analysis) z title searchtitlezcertificate of titlezland descriptionzregistered proprietor encumbranceszactivity in the last 125 dayszadministrative noticesztitle referencevolumefolio) zapplication formzplanning permitz proposed userZz for what usezdevelopment or other matterzpermit requiredzonline form applicationzplanning permit applicationc3$K|] }|vdV dSrEN.0keyword text_lowers r z'detect_document_type..s.__WT^I^_q______rc3$K|] }|vdV dSrrrs rrz'detect_document_type.. s/"g"gQX\fQf"g1"g"g"g"g"g"grr8 title_searchapplication_formr)lowersum)rtitle_search_keywordsapplication_form_keywordstitle_search_matchesapplication_form_matchesrs @rdetect_document_typer sJ!!!____0E_____""g"g"g"g4M"g"g"gggq ~ !Q &!!  8 8~ !$8 8!!yrcZ|r|sddiSd}t||kr |d|dz}t|}|dkrtj|}n1|dkrt j|}nt j| }t|\}}|rdd |iS|sdd iS |d r+tj d d|}| d }tj |}t|tsddiSd|vr||d<|S#tj$rB}t#d|t#d|ddt%|icYd}~Sd}~wt&$r}ddt%|icYd}~Sd}~wwxYw)z@Extract structured data from PDF text with proper error handlingrlzEmpty or invalid text inputi@Nr\r)rrr]r^r_r`rarzInvalid response format document_type detected_typez'JSON decoding error in PDF extraction: rbz!Failed to parse OpenAI response: z)Unexpected error in PDF data processing: )rrcrrrdr rrYr-rerfrgrhrir)r*rjrRrr)rrkrr:r@rlrrWs r!extract_structured_data_from_textr/sQ 8tzz||8677J 4yy:)KZK 5()..M&I/6DAAA , ,I3:EEE064HHH4F;;NGU <:5::;; 7566O   e $ $ ,f-r7;;GnnU++Gz'""$%% 867 7 $ & 2$1D !  GGG ;;;<<< 'g''(((ESVVEEFFFFFFF OOOMSVVMMNNNNNNNOs7;A,D3( D33F*7E?9F*? F* F%F*%F*c"d}d}tdt|}tdtj|dtdt|}tdtj|d||fS) z>Test function to validate extraction with sample document dataz TITLE SEARCH DOCUMENT Land Description: 3 Hopetoun Street NORTHCOTE VIC 3070 Registered Proprietor: John Smith and Jane Smith Encumbrances: None Activity in the last 125 days: No recent activity Administrative Notices: None a ONLINE FORM APPLICATION FOR PLANNING PERMIT Application No: D/30/2024 Date: 31 January 2024 Land Description: 3 Hopetoun Street NORTHCOTE VIC 3070 Current Use/Development: Residence Proposed Use: Residential/Accommodation Description: Two (2) Dwellings on Two (2) Lots as shown on the plans accompanying the application Estimated cost of development: $1,080,000 z"Testing Title Search extraction...zTitle Search Result: rF)indentz' Testing Application Form extraction...zApplication Form Result: )rRrrhdumps)title_search_sampleapplication_form_sample title_result app_results r test_extraction_with_sample_datarjs  .///45HIIL F$*\!"D"D"D F FGGG 455523JKKJ HdjA&F&F&F H HIII  ##rrp@c g}tdt||D]}||||z}g}|D]p} ||} || $#t$r@} t d| |dt | iYd} ~ id} ~ wwxYw||||zt|kr0t d||zdzd|dtj||S) z9Process items in batches with delays to avoid rate limitsrzError processing item: rlNzProcessed batch rEz , waiting z seconds...) rIrcappendrrRrextendrSrT) r+ process_func batch_sizedelay_between_batchesresultsibatch batch_resultsitemresultrWs rprocess_batch_with_delayrsNG 1c%jj* - -..aJ&'  8 8D 8%d++$$V,,,, 8 8 8333444$$gs1vv%677777777 8 }%%% z>CJJ & . dQ ]Q%6ddBWddd e e e J, - - - Ns A B""6BB")r8r9)rpr)+osrhpandasrrSrP django.confr django.httprdjango.views.decorators.httprshaoApp.modelsropenaishaoApp.promptsrrr rerr r r r OpenAIOPENAI_API_KEYrJrrr*r7rr.r/rYrmrrrrrrrrrrsJ  $$$$$$======111111 ~~~~~~~~~~ PPPPPPPPPPPP x6 7 7 7$4$D$$$$N%(%(3%(S%(RW%(%(%(%(P------`4 c4 d4 4 4 4 n0 C0 D0 0 0 0 f!s!s!!!!H8OC8OD8O8O8O8Ov&$&$&$Rr