updage google regions
This commit is contained in:
@@ -250,15 +250,40 @@ def curate_from_google_trends(posted_titles_data, posted_titles, used_images_dat
|
||||
try:
|
||||
logging.debug(f"Using {len(posted_titles)} posted titles and {len(used_images)} used images")
|
||||
|
||||
trends = scrape_google_trends()
|
||||
if not trends:
|
||||
logging.info("No Google Trends data available")
|
||||
# Define regions to scrape
|
||||
regions = ['US', 'GB', 'AU']
|
||||
all_trends = []
|
||||
|
||||
# Scrape trends for each region
|
||||
for geo in regions:
|
||||
logging.info(f"Scraping Google Trends for geo={geo}")
|
||||
trends = scrape_google_trends(geo=geo)
|
||||
if trends:
|
||||
logging.info(f"Collected {len(trends)} trends for geo={geo}")
|
||||
all_trends.extend(trends)
|
||||
else:
|
||||
logging.warning(f"No trends collected for geo={geo}")
|
||||
|
||||
# Remove duplicates by title and sort by search volume
|
||||
unique_trends = []
|
||||
seen_titles = set()
|
||||
for trend in all_trends:
|
||||
if trend["title"] not in seen_titles:
|
||||
unique_trends.append(trend)
|
||||
seen_titles.add(trend["title"])
|
||||
|
||||
if not unique_trends:
|
||||
logging.info("No Google Trends data available across regions")
|
||||
return None, None, False
|
||||
|
||||
# Sort trends by search volume in descending order
|
||||
unique_trends.sort(key=lambda x: x["search_volume"], reverse=True)
|
||||
logging.info(f"Total unique trends collected: {len(unique_trends)}")
|
||||
|
||||
attempts = 0
|
||||
max_attempts = 10
|
||||
while attempts < max_attempts and trends:
|
||||
trend = trends.pop(0)
|
||||
while attempts < max_attempts and unique_trends:
|
||||
trend = unique_trends.pop(0)
|
||||
title = trend["title"]
|
||||
link = trend.get("link", "")
|
||||
summary = trend.get("summary", "")
|
||||
|
||||
Reference in New Issue
Block a user