o gB@sdZddlZddlZddlZddlmZddlZddlZddl m Z ddl m Z GdddZ edkrJe ZejejZeed d ZeedSdS) Get items from dynamodbN)Decimal)Attr)DynamoConnectorc@seZdZdZdZddZddZddZd9d d Zd:d d Z de fddZ e j e jejjdd d:ddZdefddZddZddZddZddZd d!Zed"d#Z d;d%d&Zdd/e d0e d1e fd2d3Z! d9d4d5Z"d?d7d8Z#d S)@DynamoDBrslides_researchcCst}|j|_|jd|_|jd|_|jd|_|jd|_|jd|_|jd|_|jd|_ |jd|_ |jd |_ |jd |_ |jd |_ |jd |_|jd |_dS)NuserTeamcompany_information LLM_results company_listprojects projectsV2document_tables fund_infopress_releases web_pages project_docsproject_structure)rdynamodbTablerteamcompany_info_table llm_tablecompany_list_tabler rrfund_info_tablerrrr)self db_connectorr+/var/www/html/XCapMarket/utils/dynamo_db.py__init__szDynamoDB.__init__cCs|jSzGet the table name)name)rtablerrrget_table_name$szDynamoDB.get_table_namecCs |j|Sr!)rr)r table_namerrr get_table(s zDynamoDB.get_tableNcCs|jdd}d}t|jdkr&|dur&|jdd}|j||||id}n2t|jdkrP|durP|jt||d}d|vsHt|ddkrJdS|ddS|j||id}d |vr^dS|d }|S) rr AttributeNameNKeyFilterExpressionItemsItem) key_schemalenget_itemscanreq)rr#key_id sort_key_id root_key_name sort_key_nameresponseitemrrrr2,s"  zDynamoDB.get_itemcCsg}|dur |}|dur|j|d}n|j||d}||dgd|vr>|||||}||dgd|vs)|S)z>Get all items from dynamodb with an optional filter expressionNr,)r-ExpressionAttributeValuesr.LastEvaluatedKey)r3extendgetscan_with_backoff)rr#filter_expressionexpression_valuesitemsr9rrr scan_tableDs"zDynamoDB.scan_table look_up_valuecCs~|jdd}ddlm}|j|||d}|dg}d|vr=|j||||dd}||dgd|vs"|S)zQuery items from dynamodbrr'r*)KeyConditionExpressionr.r<)rEExclusiveStartKey)r0boto3.dynamodb.conditionsr+queryr4r>r=)rr#rDtable_partition_keyr+r9 documentsrrr query_items[s    zDynamoDB.query_items) max_triescCsF|dur |j|ddS|dur|j|d|dS|j|d||dS)Nr<rFrFr-)rFr-r;)r3)rr9r#r@rArrrr?sszDynamoDB.scan_with_backoffpartition_keyscCs2|j|i}|jj|d}|di|jg}|S)z?get all of the items in the table with the given partition keys RequestItems Responses)r"rbatch_get_itemr>)rr#rPkeysr9rBrrrrTs zDynamoDB.batch_get_itemcCsd}d}|dD]}|ddkr|d}|ddkr|d}q|d}d|vr3|d ks/|dur3|d}|d ks;|durK|d d d d d}||d t|dtr\|ddnd||dS)zEConvert the dynamodb items to the format outputted by the recommendorNr categoryOverviewr9 HQ_Founded company_namer"root_urlzwww..rlogologo_url)rY company_urlr^overview hq_n_founded)split capitalize isinstancedict)r company_info llm_resultrarbresultsrYrrrconvert_for_recommenders,    z DynamoDB.convert_for_recommendercCs(||j|}||j|}|||Sr)r2rrrj)rr[rgrhrrrget_item_for_recommenders z!DynamoDB.get_item_for_recommendercCs||j|}|Srk)r2r)rr[rrrrget_item_for_fund_recommendersz&DynamoDB.get_item_for_fund_recommendercsVt|trfdd|DSt|trfdd|DSt|tr)tt|S|S)Ncsi|] \}}||qSrconvert_floats_to_decimals.0kvrrr sz7DynamoDB.convert_floats_to_decimals..csg|]}|qSrrn)rqelementrtrr sz7DynamoDB.convert_floats_to_decimals..)rerfrBlistfloatrstr)rr:rrtrros    z#DynamoDB.convert_floats_to_decimalscCsH||}z |j|d}W|Sty#td|j|d}Y|Sw)zPut item into collectionr/ )roput_item Exceptiontimesleep)rr#datar9rrrupload_to_dynamodbs  zDynamoDB.upload_to_dynamodbcCsddd|DS)z5Remove non-printable characters from the text fields.rZcss|] }|r|VqdSN) isprintable)rqcrrr sz*DynamoDB._sanitize_text..)join)textrrr_sanitize_textszDynamoDB._sanitize_textc sFj|}g}|D]}fdd|D}||q tdt|dD]{} || | d} d} | |krz!|} | D]}| j|dq=Wdn1sPwYWnItj j y} zt d| d| d d |d | d 7} t |WYd} ~ nd} ~ wty} z t d | WYd} ~ n d} ~ ww| |ks5q%dS) ag Batch upload items into the DynamoDB table with retry logic. :param table_name: Name of the DynamoDB table. :param data_list: A list of dictionaries, each representing an item to upload. :param max_retries: Number of retries for unprocessed items. :param retry_delay: Time in seconds to wait before retrying. cs*i|]\}}|t|tr|n|qSr)rerzrrprtrrrusz5DynamoDB.batch_upload_to_dynamodb..rr{NzDynamoDB error: z , retrying r)/z...zUnexpected error: )rrrBappendranger1 batch_writerr}botocore exceptions ClientErrorprintrrr~)rr% data_list max_retries retry_delayr#cleaned_data_listr:sanitized_itemi batch_chunkretriesbatcherrtrbatch_upload_to_dynamodbsB     z!DynamoDB.batch_upload_to_dynamodbFc Cs||j|}|dur$|jjd|id|ttjddd|S|jj||dttjdttjddd dS) z Get or update item in collectionNr[z7set scrapped = :s, modified_by = :m, modified_date = :dz%Y-%m-%d %H:%M:%S)z:sz:mz:d)r+UpdateExpressionr;r)r[scrappedsize created_by created_date modified_by modified_dater{) r2r update_itemsocket gethostnamedatetimenowstrftimer})rr[rr9rrrcreate_or_update_company_lists4   z&DynamoDB.create_or_update_company_listcCsg}|dur |}n|j|d}|d}d|vrXz|dur'|j|dd}n |j|d|d}||d7}WntyStd|j|dd}||d7}Ynwd|vs|S)z-For a given table, get all items in the tableNr,r.r<rNrOr|)r3r~rr)rr#r@rBr9rrrget_all_table_itemss2    zDynamoDB.get_all_table_itemsc Csg}|d} |p i}||d<|r|r| d|d7} ||d<| |d} |r*|| d<|jd i| } || dgd | vrZ| d | d <|jd i| } || dgd | vs?|S) a Query items from DynamoDB based on partition key, optionally sort key, and with an optional filter expression. Args: table: DynamoDB table object. partition_key_name: The name of the partition key. partition_key_value: The value of the partition key. sort_key_name: (Optional) The name of the sort key. sort_key_value: (Optional) The value of the sort key. filter_expression: (Optional) Additional filter expression. expression_values: (Optional) Expression attribute values for the filter expression. Returns: List of items matching the query. z = :partition_keyz:partition_keyz AND z = :sort_keyz :sort_key)rEr;r-r.r<rFNrrHr=r>) rr#partition_key_namepartition_key_valuer8sort_key_valuer@rArB key_condition query_paramsr9rrr query_table>s(  zDynamoDB.query_table project_id index_namerreturncCsg}|j||d|d||id}||dgd|vrC|j||d|d||i|dd}||dgd|vs!|S)zo Query items from DynamoDB based on index name, key condition, and optional expression values. z = ::) IndexNamerEr;r.r<)rrEr;rFr)rr#rrrAsort_keyrBr9rrr query_indexus"      zDynamoDB.query_indexcCs|j||||dS)zUpdate item in collection)r+rr; ReturnValues)r)rr#keyupdate_expressionrA return_valuesrrrrs zDynamoDB.update_itemdcCspg}tdt||D]+}||||}|jj|jd|iid}d|vr5|j|dvr5||d|jq |S)a/ Batch get items from DynamoDB table using a list of keys. Args: table: DynamoDB table object keys: List of key dictionaries chunk_size: Size of each batch request (max 100) Returns: List of items retrieved from DynamoDB rKeysrQrS)rr1rrTr"r=)rr#rU chunk_sizerBrchunkr9rrrbatch_get_itemss  zDynamoDB.batch_get_itemsr)NN)rrL)F)NNNN)Nr)r)$__name__ __module__ __qualname____doc__slides_research_tabler r$r&r2rCrzrKbackoff on_exceptionexporrrr?rfrTrjrlrmror staticmethodrrrrrrxrrrrrrrrsR      . #$ <   r__main__rgreenjunkremoval)rrrrdecimalrrbotocore.exceptionsrrGrutils.dynamo_connectorrrrdbrrrr#rrBrrrrrs&   ,