import datetime from billiard.exceptions import SoftTimeLimitExceeded from django.db.models import Subquery from django.utils import timezone from verify_trusted.companies.models import Review from verify_trusted.reviews.models import ReviewSource import googlemaps APIKEY = 'AIzaSyCN53X1zvBYdVz09FJyrPDHXeWzRUgCRK8' gmaps = googlemaps.Client(key=APIKEY) headers = {'content-type': 'application/json', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'} class GoogleMapCrawler: # https://github.com/googlemaps/google-maps-services-python # https://googlemaps.github.io/google-maps-services-python/docs/index.html # @staticmethod # def get_place_id(place_name, place_location, company_address): # place_location = place_location + " " + company_address # places_result = gmaps.places(place_name, location=place_location) # print("list places") # if len(places_result['results'])>0: # for place in places_result['results']: # if place['formatted_address']==company_address: # return place['place_id'] # else: # return None # return None # # # def get_reviews(self, url, company_name, company_address): # try: # print(url) # print(company_name) # res = requests.get(url, headers=headers) # # res = requests.get(url) # # soup = BeautifulSoup(res.text, 'html.parser') # # print(res.text) # # place_name = soup.find("meta", property="og:title")['content'].replace("ยท", "").replace("Google Maps", "").strip() # # if place_name == "": # # place_name = res.url.split("/")[-3] # # print(place_name) # place_location = res.url.split("/")[-2] # print(place_location) # # print(place_location) # # addition_info = res.url.split("/")[-1] # place_id = self.get_place_id(company_name, place_location, company_address) # if place_id is not None: # place = gmaps.place(place_id=place_id) # print('================================================================================================') # print(place['result']) # print('================================================================================================') # total_reviews = place['result']['user_ratings_total'] # avg_ratings = place['result']['rating'] # reviews = place['result']['reviews'] # parsed_reviews = [ # { # 'id': None, # 'author': review['author_name'], # 'date': datetime.datetime.date(datetime.datetime.fromtimestamp(review['time'])), # 'headline': '', # 'body': review['text'], # 'rating': review['rating'], # } # for review in reviews # ] # print(f"{total_reviews}, {avg_ratings}, {len(parsed_reviews)}") # print(parsed_reviews) # return total_reviews, avg_ratings, parsed_reviews # else: # return None # except Exception as e : # print(e) # return None def get_reviews(self, place_id): try: place = gmaps.place(place_id=place_id) print('================================================================================================') print(place['result']) print('================================================================================================') total_reviews = place['result']['user_ratings_total'] avg_ratings = place['result']['rating'] reviews = place['result']['reviews'] url = place['result']['url'] parsed_reviews = [ { 'id': None, 'author': review['author_name'], 'date': datetime.datetime.date(datetime.datetime.fromtimestamp(review['time'])), 'headline': '', 'body': review['text'], 'rating': review['rating'], } for review in reviews ] print(f"{total_reviews}, {avg_ratings}, {len(parsed_reviews)}") print(parsed_reviews) return total_reviews, avg_ratings, parsed_reviews, url except Exception as e: print(e) return None def sync_reviews(self, review_source: ReviewSource): try: company_map = review_source.company.map print(company_map) place_id = company_map["place_id"] # review_source.url = result = self.get_reviews(place_id) if result is None: print("Result is None") review_source.sync_status = ReviewSource.SyncStatus.FALSE review_source.save() return total_reviews, avg_rating, reviews, url = result review_source.url = url review_source.save() print( "================================SYNC REVIEWS=========================================================") reviews = [ Review( source=review_source, author=r['author'], date=r['date'], headline=r['headline'], body=r['body'], rating=r['rating'], date_parse=timezone.now(), external_id=r['id'], ) for r in reviews ] print("start insert") uncheck_reviews = Review.objects.bulk_create(reviews, ignore_conflicts=False) # sua lai khi test xong print("end insert") print(uncheck_reviews) try: print("start check dup reviews") dup_reviews_ids = [r.id for r in uncheck_reviews] for row in uncheck_reviews: reviews = Review.objects.filter(source_id=review_source.id, author=row.author, date=row.date).exclude( id__in=list(dup_reviews_ids)) reviews.delete() active_reviews = Review.objects.filter(source_id__in=Subquery(ReviewSource.objects.filter( company_id=review_source.company_id).values_list('id', flat=True)), is_active=True).order_by( 'display_order', '-date_modify', '-date') num_active_reviews = len(active_reviews) if num_active_reviews < 20: nested_q = Review.objects.filter(source_id__in=Subquery(ReviewSource.objects.filter( company_id=review_source.company_id).values_list('id', flat=True)), is_active=False).order_by( 'display_order', '-date_modify', '-date')[:(20 - num_active_reviews)] Review.objects.filter(pk__in=nested_q).update(is_active=True) elif num_active_reviews > 20: nested_q = Review.objects.filter(source_id__in=Subquery(ReviewSource.objects.filter( company_id=review_source.company_id).values_list('id', flat=True)), is_active=True).order_by('display_order', '-date_modify', '-date')[ 20:] Review.objects.filter(pk__in=nested_q).update(is_active=False, display_order=None) # widget = Widget.objects.filter(company=review_source.company_id).first() # widget_reviews = WidgetReivews.objects.filter(widget=widget.id).values_list('review', flat=True) # num_widget_reviews = len(widget_reviews) # ws = [] # if num_widget_reviews < 5: # nested_w = Review.objects.filter(source_id__in=Subquery(ReviewSource.objects.filter( # company_id=review_source.company_id,platform__status=Platform.Status.ACTIVE).values_list('id', flat=True))).order_by('display_order', '-date_modify', '-date')[:(5-num_widget_reviews)] # for w in nested_w: # ws.append(WidgetReivews(widget=widget, review=w)) # WidgetReivews.objects.bulk_create(ws, ignore_conflicts=False) # elif num_active_reviews > 5: # pass review_source.sync_status = ReviewSource.SyncStatus.SYNCED print(f"{datetime.datetime.now()} - {total_reviews} : {avg_rating}") review_source.reviews_count = total_reviews print(f"{datetime.datetime.now()} - set total reviews") review_source.average_rating = avg_rating print(f"{datetime.datetime.now()} - set avg rating") review_source.save() print(f"done") except SoftTimeLimitExceeded as e: review_source.sync_status = ReviewSource.SyncStatus.FALSE review_source.save() print(e) print('Check reviews false') except Exception as e: review_source.sync_status = ReviewSource.SyncStatus.FALSE review_source.save() print(e) print('Sync reviews false')