Ë
    |v…ji3  ã                   ó¢  — d Z ddlZddlZddlmZ ddlmZmZ dZ	dZ
dZg d¢Zd	d
gZ ee«      j                  dz  dz  ZddgZdedee   fd„Ze G d„ d«      «       Zdee   dee   dedz  fd„Zdedz  dee   fd„Zdedz  dee   fd„Zdee   dee   deeedz  edz  f      fd„Zdddddedfd eez  eeez     z  d!edz  d"edz  d#ed$ed%ed&ed'edz  defd(„Zed)k(  rîddlZddl Z  e!ejD                  «      d*kD  rejD                  d*   nd+Z# ee#«      Z$ e% e jL                  e$jN                  d,¬-«      «        e%d.«        e%e$jP                  jS                  g d/¢«      jU                  d0«      «        e%d1«       e$jV                  jX                  r3 e%e$jV                  jS                  g d2¢«      jU                  d0«      «       y e%d3«       yy)4a}  
Lead data cleaning engine.

Design goals:
- Handle millions of rows without loading everything into memory unnecessarily
  (Polars lazy/streaming where practical).
- Never silently corrupt data: every rejected row is logged with a reason.
- Be delimiter-agnostic for multi-value fields by extracting values via
  pattern matching rather than assuming a fixed separator character.
é    N)ÚPath)Ú	dataclassÚfieldz0[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}z \s*[:;|]\s*|\s*,\s*(?=\+|\d{3,})é   )z\.edu$z\.edu\.[a-z]{2,3}$z\.ac\.[a-z]{2,3}$z\.ac$z\.mil$z\.mil\.[a-z]{2,3}$Ú
blocklistszmilitary_domains.txtz\.myshopify\.com$z\.myshopify\.io$ÚpathÚreturnc                 ó>  — | j                  «       s
t        «       S t        | dd¬«      5 }|D �ch c]O  }|j                  «       r=|j                  «       j	                  d«      s|j                  «       j                  «       ’ŒQ c}cd d d «       S c c}w # 1 sw Y   y xY w)NÚrzutf-8)ÚencodingÚ#)ÚexistsÚsetÚopenÚstripÚ
startswithÚlower)r   ÚfÚlines      ú./var/www/html/lead-cleaner/backend/cleaning.pyÚ_load_blocklistr   5   s~   € Ø�;‰;Œ=Ü‹uˆÜ	ˆd�C 'Õ	*¨añ ó
á�Ø�z‰zŒ| D§J¡J£L×$;Ñ$;¸CÔ$@ð �J‰J‹L×ÑÕ Øñ
÷ 
+Ñ	*ùò
÷ 
+Ð	*ús   ©B®ABÂBÂBÂBc                   óz   — e Zd ZU ej                  ed<   ej                  ed<    ee¬«      Zeed<   dZ	e
dz  ed<   y)ÚCleaningResultÚclean_dfÚrejected_df)Údefault_factoryÚstatsNÚcategory_column)Ú__name__Ú
__module__Ú__qualname__ÚplÚ	DataFrameÚ__annotations__r   Údictr   r   Ústr© ó    r   r   r   D   s3   … à�l‰lÓØ—‘ÓÙ¨Ô-€Eˆ4Ó-Ø"&€O�S˜4‘ZÔ&r(   r   ÚcolumnsÚkeywordsc                 óœ   — | D �ci c]  }|j                  «       |“Œ }}|D ]$  }|j                  «       D ]  \  }}||v sŒ|c c S  Œ& y c c}w ©N)r   Úitems)r)   r*   ÚcÚ	lower_mapÚkwÚlcÚorigs          r   Ú_detect_columnr3   P   sU   € Ù'.Ó/¡w !�—‘“˜A‘ w€IÐ/ÛˆØ!Ÿ™Ö)‰HˆB�Ø�RŠxØ”ñ *ð ð ùò 0s   …A	Úrawc                 ó¼   — | sg S t        j                  t        | «      }g }|D ]6  }|j                  «       j	                  «       }||vsŒ&|j                  |«       Œ8 |S r,   )ÚreÚfindallÚEMAIL_REGEXr   r   Úappend)r4   ÚfoundÚseenÚes       r   Ú_extract_emailsr=   ]   sT   € ÙØˆ	Ü�J‰J”{ CÓ(€EØ€DÛˆØ�G‰G‹I�O‰OÓˆØ�DŠ=Ø�K‰K˜�Nð ð €Kr(   c                 óô   — | sg S t        j                  t        | «      }g }|D ]R  }|j                  «       }t        j                  dd|«      }t        |«      t        k\  sŒ=||vsŒB|j                  |«       ŒT |S )Nz\DÚ )r6   ÚsplitÚPHONE_SPLIT_REGEXr   ÚsubÚlenÚPHONE_MIN_DIGITSr9   )r4   ÚpartsÚoutÚpÚdigitss        r   Ú_extract_phonesrI   i   sl   € ÙØˆ	Ü�H‰HÔ&¨Ó,€EØ
€CÛˆØ�G‰G‹IˆÜ—‘˜˜r 1Ó%ˆÜˆv‹;Ô*Ó*¨q¸ª|Ø�J‰J�q�Mð	 ð
 €Jr(   ÚemailsÚphonesc                 óˆ  — t        | «      t        |«      }}|dk(  r|dk(  rdgS |dk  r |dkD  r| r| d   nd}|D �cg c]  }||f‘Œ c}S |dk  r |dkD  r|r|d   nd}| D �cg c]  }||f‘Œ c}S t        ||d«      }g }	t        |«      D ]-  }
|
|k  r| |
   nd}|
|k  r||
   nd}|	j                  ||f«       Œ/ |	S c c}w c c}w )a  Pair emails/phones by position. If one side has exactly one value and
    the other has several, repeat the single value across all rows. If both
    have multiple values of different lengths, pair by index and leave the
    shorter side's extra slots as None.r   )NNé   N)rC   ÚmaxÚranger9   )rJ   rK   Ún_eÚn_pÚe_valrG   Úp_valr<   ÚnÚpairsÚis              r   Ú_pair_multivaluerW   v   sô   € ô
 �6‹{œC ›Kˆ€Cà
ˆa‚x�C˜1’HØˆ~Ðà
ˆa‚x�C˜!’GÙ#��q’	¨ˆÙ$*Ó+¡F˜q�˜’
 FÑ+Ð+à
ˆa‚x�C˜!’GÙ#��q’	¨ˆÙ$*Ó+¡F˜q��E’
 FÑ+Ð+äˆC��aÓ€AØ€EÜ�1ŽXˆØ˜SšˆF�1ŠI dˆØ˜SšˆF�1ŠI dˆØ�‰�a˜�VÕð ð €Lùò ,ùò ,s   »B:Á B?TÚemailFÚ
input_pathÚemail_columnÚphone_columnÚexplode_phonesÚ
dedupe_keyÚexclude_placeholder_storesÚmilitary_blocklist_pathr   c           
      ó¬  ‡1— t        | t        t        f«      r| gn
t        | «      }t	        |«      dk(  rt        j                  |d   dd¬«      }	n<|D �
cg c]  }
t        j                  |
dd¬«      ‘Œ }}
t        j                  |d¬«      }	|	j                  }|	j                  }|xs t        |dg«      }|xs t        |d	g«      }|€t        d
«      ‚|}|€B|D �ci c]  }|j                  «       |“Œ }}|j                  d«      xs |j                  d«      }t        |«      Š1|	j                  d«      }	|	j!                  t        j"                  d«      j%                  t
        j&                  «      «      }	|	j                  }|	j)                  |D �cg c]  }|‘Œ c}dd¬«      }	||	j                  z
  }|	|   j+                  «       }|r|r|	|   j+                  «       nd g|	j                  z  }g }t-        |	j                  «      D ]W  }t/        ||   «      }|rt1        ||   «      n||   r||   gng }t3        ||«      }|D ]  \  }}
|j5                  |||
f«       Œ ŒY t        j6                  |t
        j&                  t
        j8                  t
        j8                  dœd¬«      }|	j;                  |g|r|gng z   «      }|j=                  |dd¬«      }|j?                  d|i«      }|r|j?                  d|i«      }n|j;                  d«      }|j                  } dt        d z  dt        fˆ1fd„}!|j!                  t        j"                  |«      jA                  |!t
        j8                  d¬«      jC                  d«      «      }t        |dg«      }"|"rù|"|j                  v rëdt        d z  dtD        fd„}#|j!                  t        j"                  |"«      jA                  |#t
        jF                  d¬«      jC                  d«      «      }|rT|jI                  t        j"                  d«      «      j                  }$|jI                  t        j"                  d«       «      }n1|jI                  t        j"                  d«      «      j                  }$nd}$|jI                  t        j"                  d«      jK                  d d!g«      «      }%|jI                  t        j"                  d«      jK                  d d!g«       «      }&|%jI                  t        j"                  d«      d k(  «      j                  }'|%jI                  t        j"                  d«      d!k(  «      j                  }(|&jI                  t        j"                  d«      d"k(  «      j                  })|&j;                  d«      }&|&j                  }*|dk(  r‘|&jI                  t        j"                  |«      jM                  «       «      }+|&jI                  t        j"                  |«      jO                  «       «      },|+j)                  |gdd¬«      }+t        j                  |+|,g«      }&n|dk(  r|"r|&j)                  |"gdd¬«      }&|*|&j                  z
  }-|&j;                  dd¬#«      }&|%j;                  dd¬#«      }%||| |'|(|)|-|$|&j                  d$œ	}.|r�||&j                  v r�|&jQ                  |«      jS                  t        j                  «       jC                  d%«      «      jU                  d%d¬&«      }/|/jW                  «       D �0ci c]  }0|0|   �|0|   nd'|0d%   “Œ c}0|.d(<   tY        |&|%|.|¬)«      S c c}
w c c}w c c}w c c}0w )*NrM   r   i'  T)Úinfer_schema_lengthÚignore_errorsÚdiagonal_relaxed)ÚhowrX   Úphonez?Could not detect an email column. Pass email_column explicitly.ÚplatformÚcategoryÚ_orig_row_idÚfirst)ÚsubsetÚkeepÚmaintain_order)rh   Ú_clean_emailÚ_clean_phoneÚrow)ÚschemaÚorientÚleft)Úonrd   rm   rn   r	   c                 óð   •— | rd| vry| j                  dd«      d   j                  «       }t        D ]  }t        j                  ||«      sŒ y |‰v ryt
        D ]  }t        j                  ||«      sŒ y y)NÚ@Úno_emailrM   éÿÿÿÿÚmilitary_domainÚeducational_domainÚok)Úrsplitr   ÚMIL_PATTERNSr6   ÚsearchÚEDU_PATTERNS)rX   ÚdomainÚpatÚmil_blocklists      €r   Úclassifyzclean_leads.<locals>.classifyé   sv   ø€ Ù˜ 5Ñ(ØØ—‘˜c 1Ó% bÑ)×/Ñ/Ó1ˆßˆCÜ�y‰y˜˜fÕ%Ù(ð  ð �]Ñ"Ø$ßˆCÜ�y‰y˜˜fÕ%Ù+ð  ð r(   F)Úreturn_dtypeÚ
skip_nullsÚ_reject_reasonr   Údc                 óZ   ‡ — ‰ sy‰ j                  «       Š t        ˆ fd„t        D «       «      S )NFc              3   óJ   •K  — | ]  }t        j                  |‰«      –— Œ y ­wr,   )r6   r}   )Ú.0r€   r†   s     €r   Ú	<genexpr>z6clean_leads.<locals>.is_placeholder.<locals>.<genexpr>  s   øè ø€ ÐOÑ4N¨S”r—y‘y  a×(Ñ4Nùs   ƒ #)r   ÚanyÚPLACEHOLDER_STORE_PATTERNS)r†   s   `r   Úis_placeholderz#clean_leads.<locals>.is_placeholder   s&   ø€ ÙØØ—‘“	ˆAÜÓOÕ4NÓOÓOÐOr(   Úis_placeholder_storerx   ry   rv   )Ústrict)	Úrows_in_original_fileÚexact_duplicate_rows_removedÚrows_after_email_phone_explodeÚrejected_military_domainÚrejected_educational_domainÚrows_with_no_emailÚduplicate_emails_removedÚplaceholder_stores_flaggedÚfinal_clean_row_countÚcount)Ú
descendingz(uncategorized)Úcategory_counts)r   r   r   r   )-Ú
isinstancer&   r   ÚlistrC   r"   Úread_csvÚconcatr)   Úheightr3   Ú
ValueErrorr   Úgetr   Úwith_row_indexÚwith_columnsÚcolÚcastÚInt64ÚuniqueÚto_listrO   r=   rI   rW   r9   r#   ÚStringÚdropÚjoinÚrenameÚmap_elementsÚaliasÚboolÚBooleanÚfilterÚis_inÚis_not_nullÚis_nullÚgroup_byÚaggÚsortÚto_dictsr   )2rY   rZ   r[   r\   r]   r^   r_   r   Úinput_pathsÚdfrG   ÚframesÚoriginal_columnsÚn_inputÚ	email_colÚ	phone_colÚcat_colr.   Úexact_lowerÚn_before_exact_dedupeÚn_exact_dupes_removedÚemail_seriesÚphone_seriesÚexploded_rowsrV   rJ   rK   rU   r<   Úexp_dfÚbaseÚresultÚn_after_exploder‚   Ú
domain_colr�   Ún_placeholderr   r   Ú
n_militaryÚn_eduÚ
n_no_emailÚn_before_final_dedupeÚ	has_emailÚno_email_rowsÚn_final_dupes_removedr   Úcountsro   r�   s2                                                    @r   Úclean_leadsrÖ   •   s°  ø€ ô #-¨Z¼#¼t¸Ô"E�:‘,Ì4ÐPZÓK[€Kä
ˆ;Ó˜1ÒÜ�[‰[˜ Q™¸UÐRVÔW‰ñ ZeÓeÑYdÐTU”"—+‘+˜a°UÈ$ÖOÐYdˆÐeÜ�Y‰Y�vÐ#5Ô6ˆà—z‘zÐØ�i‰i€GàÒK¤Ð/?À'ÀÓ K€IØÒK¤Ð/?À'ÀÓ K€IàÐÜÐZÓ[Ð[ð
 €GØ€Ù-=Ó>Ñ-=¨�q—w‘w“y !‘|Ð-=ˆÐ>Ø—/‘/ *Ó-ÒL°·±ÀÓ1Lˆä#Ð$;Ó<€Mð 
×	Ñ	˜>Ó	*€Bð
 
�‰œŸ™ Ó/×4Ñ4´R·X±XÓ>Ó	?€BØŸI™IÐØ	�‰Ñ&6Ó7Ñ&6 š1Ð&6Ñ7¸gÐVZˆÓ	[€BØ1°B·I±IÑ=Ðð �i‘=×(Ñ(Ó*€LÙ/8¹^�2�i‘=×(Ñ(Ô*ÐSWÐRXÐ[]×[dÑ[dÑRd€Là€MÜ�2—9‘9ÖˆÜ  ¨a¡Ó1ˆÙ5C” ¨a¡Ô1Ø!-¨a¢ˆ\˜!‰_Ñ°bð 	ô ! ¨Ó0ˆÛ‰DˆAˆqØ× Ñ  ! Q¨ Õ+ñ ð ô �\‰\ØÜ "§¡¼"¿)¹)ÔUW×U^ÑU^Ñ_Øô€Fð �7‰7�I�;± 9¡+ÀÑCÓD€DØ�[‰[˜ .°fˆ[Ó=€FØ�]‰]˜N¨IÐ6Ó7€FÙØ—‘ °	Ð:Ó;‰à—‘˜^Ó,ˆà—m‘m€Oðœ˜d™
ð ¤sõ ð × Ñ Ü
�‰ˆyÓß	‰�h¬R¯Y©YÀ5ˆÓ	Iß	‰ÐÓ	 ó€Fô  Ð 0°8°*Ó=€JÙ�j F§N¡NÑ2ð	Pœc D™jð 	P¬Tó 	Pð ×$Ñ$Ü�F‰F�:Óß‰\˜.´r·z±zÈeˆ\ÓTß‰UÐ)Ó*ó
ˆñ
 &Ø"ŸM™M¬"¯&©&Ð1GÓ*HÓI×PÑPˆMØ—]‘]¤B§F¡FÐ+AÓ$BÐ#BÓC‰Fà"ŸM™M¬"¯&©&Ð1GÓ*HÓI×PÑP‰Màˆð —-‘-¤§¡Ð'7Ó 8× >Ñ >Ð@QÐSgÐ?hÓ iÓj€KØ�}‰}œbŸf™fÐ%5Ó6×<Ñ<Ð>OÐQeÐ=fÓgÐgÓh€Hà×#Ñ#¤B§F¡FÐ+;Ó$<Ð@QÑ$QÓR×YÑY€JØ×ÑœrŸv™vÐ&6Ó7Ð;OÑOÓP×WÑW€EØ—‘¤§¡Ð(8Ó!9¸ZÑ!GÓH×OÑO€Jà�}‰}Ð-Ó.€Hð %ŸO™OÐØ�WÒà—O‘O¤B§F¡F¨9Ó$5×$AÑ$AÓ$CÓDˆ	Ø Ÿ™¬¯©¨yÓ(9×(AÑ(AÓ(CÓDˆØ×$Ñ$¨Y¨K¸gÐVZÐ$Ó[ˆ	Ü—9‘9˜i¨Ð7Ó8‰Ø	�xÒ	¡JØ—?‘?¨:¨,¸WÐUY�?ÓZˆð 2°H·O±OÑCÐà�}‰}˜^°Eˆ}Ó:€HØ×"Ñ" >¸%Ð"Ó@€Kð ")Ø(=Ø*9Ø$.Ø',Ø(Ø$9Ø&3Ø!)§¡ñ
€Eñ �7˜h×.Ñ.Ñ.à×Ñ˜gÓ&ß‰S”—‘“—‘ Ó(Ó)ß‰T�' dˆTÓ+ð 	ð —‘Ô(ó$
á(�ð ! ™\Ð5ˆS�Š\Ð;LÈsÐSZÉ|Ñ[Ø(ñ$
ˆÐÑ ô
  8¸ÈEÐcjÔkÐkùòy fùò" ?ùò 8ùòr$
s   Á]Ã]Æ	]Ü]Ú__main__rM   z
sample.csvé   )Úindentz
--- clean sample ---)r   rJ   rK   rf   é
   z
--- rejected sample ---)r   rJ   r…   z(none in this file))-Ú__doc__r6   Úpolarsr"   Úpathlibr   Údataclassesr   r   r8   rA   rD   r~   r|   Ú__file__ÚparentÚDEFAULT_MILITARY_BLOCKLIST_PATHrŒ   r   r&   r   r   r�   r3   r=   rI   ÚtuplerW   r°   rÖ   r   ÚsysÚjsonrC   Úargvr   ÚresÚprintÚdumpsr   r   ÚselectÚheadr   r    r'   r(   r   Ú<module>rë      s¡  ðñ	ó 
Û Ý ß (ð B€ð 8Ð ØÐ ò€ð Øð€ñ
 #' x£.×"7Ñ"7¸,Ñ"FÐI_Ñ"_Ð ð ØðÐ ð
˜$ð 
 3 s¡8ó 
ð ÷'ð 'ó ð'ð˜D ™Ið °°c±ð ¸sÀT¹zó ð	˜˜t™ð 	¨¨S©	ó 	ð
˜˜t™ð 
¨¨S©	ó 
ð˜T #™Yð °°S±	ð ¸dÀ5ÈÈtÉÐUXÐ[_ÑU_ÐI_ÑC`Ñ>aó ðB  $Ø#ØØØ',Ø$CØ"&ñplØ�d‘
˜T #¨¡*Ñ-Ñ-ðplà˜‘*ðplð ˜‘*ðplð ð	plð
 ðplð !%ðplð "ðplð ˜4‘Zðplð óplðf ˆzÒßÙ˜cŸh™h›-¨!Ò+ˆ3�8‰8�AŠ;°€DÙ
�dÓ
€CÙ	ˆ*ˆ$�*‰*�S—Y‘Y qÔ
)Ô*Ù	Ð
"Ô#Ù	ˆ#�,‰,×
Ñ
ÒHÓ
I×
NÑ
NÈrÓ
RÔSÙ	Ð
%Ô&Ø
‡�×ÒÙˆc�o‰o×$Ñ$Ò%KÓL×QÑQÐRTÓUÕVáÐ#Õ$ð r(   