|
|
|
|
@ -250,15 +250,40 @@ def curate_from_google_trends(posted_titles_data, posted_titles, used_images_dat |
|
|
|
|
try: |
|
|
|
|
logging.debug(f"Using {len(posted_titles)} posted titles and {len(used_images)} used images") |
|
|
|
|
|
|
|
|
|
trends = scrape_google_trends() |
|
|
|
|
if not trends: |
|
|
|
|
logging.info("No Google Trends data available") |
|
|
|
|
# Define regions to scrape |
|
|
|
|
regions = ['US', 'GB', 'AU'] |
|
|
|
|
all_trends = [] |
|
|
|
|
|
|
|
|
|
# Scrape trends for each region |
|
|
|
|
for geo in regions: |
|
|
|
|
logging.info(f"Scraping Google Trends for geo={geo}") |
|
|
|
|
trends = scrape_google_trends(geo=geo) |
|
|
|
|
if trends: |
|
|
|
|
logging.info(f"Collected {len(trends)} trends for geo={geo}") |
|
|
|
|
all_trends.extend(trends) |
|
|
|
|
else: |
|
|
|
|
logging.warning(f"No trends collected for geo={geo}") |
|
|
|
|
|
|
|
|
|
# Remove duplicates by title and sort by search volume |
|
|
|
|
unique_trends = [] |
|
|
|
|
seen_titles = set() |
|
|
|
|
for trend in all_trends: |
|
|
|
|
if trend["title"] not in seen_titles: |
|
|
|
|
unique_trends.append(trend) |
|
|
|
|
seen_titles.add(trend["title"]) |
|
|
|
|
|
|
|
|
|
if not unique_trends: |
|
|
|
|
logging.info("No Google Trends data available across regions") |
|
|
|
|
return None, None, False |
|
|
|
|
|
|
|
|
|
# Sort trends by search volume in descending order |
|
|
|
|
unique_trends.sort(key=lambda x: x["search_volume"], reverse=True) |
|
|
|
|
logging.info(f"Total unique trends collected: {len(unique_trends)}") |
|
|
|
|
|
|
|
|
|
attempts = 0 |
|
|
|
|
max_attempts = 10 |
|
|
|
|
while attempts < max_attempts and trends: |
|
|
|
|
trend = trends.pop(0) |
|
|
|
|
while attempts < max_attempts and unique_trends: |
|
|
|
|
trend = unique_trends.pop(0) |
|
|
|
|
title = trend["title"] |
|
|
|
|
link = trend.get("link", "") |
|
|
|
|
summary = trend.get("summary", "") |
|
|
|
|
|