updage google regions
This commit is contained in:
@@ -250,15 +250,40 @@ def curate_from_google_trends(posted_titles_data, posted_titles, used_images_dat
|
|||||||
try:
|
try:
|
||||||
logging.debug(f"Using {len(posted_titles)} posted titles and {len(used_images)} used images")
|
logging.debug(f"Using {len(posted_titles)} posted titles and {len(used_images)} used images")
|
||||||
|
|
||||||
trends = scrape_google_trends()
|
# Define regions to scrape
|
||||||
if not trends:
|
regions = ['US', 'GB', 'AU']
|
||||||
logging.info("No Google Trends data available")
|
all_trends = []
|
||||||
|
|
||||||
|
# Scrape trends for each region
|
||||||
|
for geo in regions:
|
||||||
|
logging.info(f"Scraping Google Trends for geo={geo}")
|
||||||
|
trends = scrape_google_trends(geo=geo)
|
||||||
|
if trends:
|
||||||
|
logging.info(f"Collected {len(trends)} trends for geo={geo}")
|
||||||
|
all_trends.extend(trends)
|
||||||
|
else:
|
||||||
|
logging.warning(f"No trends collected for geo={geo}")
|
||||||
|
|
||||||
|
# Remove duplicates by title and sort by search volume
|
||||||
|
unique_trends = []
|
||||||
|
seen_titles = set()
|
||||||
|
for trend in all_trends:
|
||||||
|
if trend["title"] not in seen_titles:
|
||||||
|
unique_trends.append(trend)
|
||||||
|
seen_titles.add(trend["title"])
|
||||||
|
|
||||||
|
if not unique_trends:
|
||||||
|
logging.info("No Google Trends data available across regions")
|
||||||
return None, None, False
|
return None, None, False
|
||||||
|
|
||||||
|
# Sort trends by search volume in descending order
|
||||||
|
unique_trends.sort(key=lambda x: x["search_volume"], reverse=True)
|
||||||
|
logging.info(f"Total unique trends collected: {len(unique_trends)}")
|
||||||
|
|
||||||
attempts = 0
|
attempts = 0
|
||||||
max_attempts = 10
|
max_attempts = 10
|
||||||
while attempts < max_attempts and trends:
|
while attempts < max_attempts and unique_trends:
|
||||||
trend = trends.pop(0)
|
trend = unique_trends.pop(0)
|
||||||
title = trend["title"]
|
title = trend["title"]
|
||||||
link = trend.get("link", "")
|
link = trend.get("link", "")
|
||||||
summary = trend.get("summary", "")
|
summary = trend.get("summary", "")
|
||||||
|
|||||||
Reference in New Issue
Block a user