update realtime rate limit for X
This commit is contained in:
+51
-19
@@ -24,10 +24,12 @@ from foodie_config import (
|
||||
)
|
||||
from foodie_utils import (
|
||||
load_json_file, save_json_file, get_image, generate_image_query,
|
||||
upload_image_to_wp, select_best_persona, determine_paragraph_count,
|
||||
upload_image_to_wp, determine_paragraph_count, insert_link_naturally,
|
||||
is_interesting, generate_title_from_summary, summarize_with_gpt4o,
|
||||
generate_category_from_summary, post_to_wp, prepare_post_data,
|
||||
smart_image_and_filter, insert_link_naturally, get_flickr_image
|
||||
select_best_author, smart_image_and_filter, get_flickr_image,
|
||||
get_next_author_round_robin, fetch_duckduckgo_news_context,
|
||||
check_author_rate_limit
|
||||
)
|
||||
from foodie_hooks import get_dynamic_hook, get_viral_share_prompt
|
||||
from dotenv import load_dotenv
|
||||
@@ -246,47 +248,61 @@ def fetch_duckduckgo_news_context(trend_title, hours=24):
|
||||
logging.error(f"Failed to fetch DuckDuckGo News context for '{trend_title}' after {MAX_RETRIES} attempts")
|
||||
return trend_title
|
||||
|
||||
def curate_from_google_trends(geo_list=['US']):
|
||||
def curate_from_google_trends():
|
||||
try:
|
||||
all_trends = []
|
||||
for geo in geo_list:
|
||||
trends = scrape_google_trends(geo=geo)
|
||||
if trends:
|
||||
all_trends.extend(trends)
|
||||
|
||||
if not all_trends:
|
||||
global posted_titles_data, posted_titles, used_images
|
||||
posted_titles_data = load_json_file(POSTED_TITLES_FILE, EXPIRATION_HOURS)
|
||||
posted_titles = set(entry["title"] for entry in posted_titles_data)
|
||||
used_images = set(entry["title"] for entry in load_json_file(USED_IMAGES_FILE, IMAGE_EXPIRATION_DAYS) if "title" in entry)
|
||||
logging.debug(f"Loaded {len(posted_titles)} posted titles and {len(used_images)} used images")
|
||||
|
||||
trends = fetch_google_trends()
|
||||
if not trends:
|
||||
print("No Google Trends data available")
|
||||
logging.info("No Google Trends data available")
|
||||
return None, None, False
|
||||
|
||||
|
||||
attempts = 0
|
||||
max_attempts = 10
|
||||
while attempts < max_attempts and all_trends:
|
||||
trend = all_trends.pop(0)
|
||||
while attempts < max_attempts and trends:
|
||||
trend = trends.pop(0)
|
||||
title = trend["title"]
|
||||
link = trend.get("link", "https://trends.google.com/")
|
||||
link = trend.get("link", "")
|
||||
summary = trend.get("summary", "")
|
||||
source_name = "Google Trends"
|
||||
source_name = trend.get("source", "Google Trends")
|
||||
original_source = f'<a href="{link}">{source_name}</a>'
|
||||
|
||||
if title in posted_titles:
|
||||
print(f"Skipping already posted trend: {title}")
|
||||
logging.info(f"Skipping already posted trend: {title}")
|
||||
attempts += 1
|
||||
continue
|
||||
|
||||
print(f"Trying Google Trend: {title} from {source_name}")
|
||||
logging.info(f"Trying Google Trend: {title} from {source_name}")
|
||||
|
||||
image_query, relevance_keywords, main_topic, skip = smart_image_and_filter(title, summary)
|
||||
try:
|
||||
image_query, relevance_keywords, main_topic, skip = smart_image_and_filter(title, summary)
|
||||
except Exception as e:
|
||||
print(f"Smart image/filter error for '{title}': {e}")
|
||||
logging.warning(f"Failed to process smart_image_and_filter for '{title}': {e}")
|
||||
attempts += 1
|
||||
continue
|
||||
|
||||
if skip:
|
||||
logging.info(f"Skipping filtered Google Trend: {title}")
|
||||
print(f"Skipping filtered trend: {title}")
|
||||
logging.info(f"Skipping filtered trend: {title}")
|
||||
attempts += 1
|
||||
continue
|
||||
|
||||
ddg_context = fetch_duckduckgo_news_context(title)
|
||||
scoring_content = f"{title}\n\n{summary}\n\nAdditional Context: {ddg_context}"
|
||||
interest_score = is_interesting(scoring_content)
|
||||
print(f"Interest Score for '{title[:50]}...': {interest_score}")
|
||||
logging.info(f"Interest score for '{title}': {interest_score}")
|
||||
if interest_score < 6:
|
||||
logging.info(f"Google Trends Interest Too Low: {interest_score}")
|
||||
print(f"Trend Interest Too Low: {interest_score}")
|
||||
logging.info(f"Trend Interest Too Low: {interest_score}")
|
||||
attempts += 1
|
||||
continue
|
||||
|
||||
@@ -308,6 +324,7 @@ def curate_from_google_trends(geo_list=['US']):
|
||||
extra_prompt=extra_prompt
|
||||
)
|
||||
if not final_summary:
|
||||
print(f"Summary failed for '{title}'")
|
||||
logging.info(f"Summary failed for '{title}'")
|
||||
attempts += 1
|
||||
continue
|
||||
@@ -329,15 +346,17 @@ def curate_from_google_trends(geo_list=['US']):
|
||||
category = post_data["categories"][0]
|
||||
image_url, image_source, uploader, page_url = get_flickr_image(image_query, relevance_keywords, main_topic)
|
||||
if not image_url:
|
||||
print(f"Flickr image fetch failed for '{image_query}', trying fallback")
|
||||
logging.warning(f"Flickr image fetch failed for '{image_query}', trying fallback")
|
||||
image_url, image_source, uploader, page_url = get_image(image_query)
|
||||
if not image_url:
|
||||
print(f"All image uploads failed for '{title}' - posting without image")
|
||||
logging.warning(f"All image uploads failed for '{title}' - posting without image")
|
||||
image_source = None
|
||||
uploader = None
|
||||
page_url = None
|
||||
|
||||
hook = get_dynamic_hook(post_data["title"]).strip()
|
||||
|
||||
share_prompt = get_viral_share_prompt(post_data["title"], final_summary)
|
||||
share_links_template = (
|
||||
f'<p>{share_prompt} '
|
||||
@@ -362,7 +381,13 @@ def curate_from_google_trends(geo_list=['US']):
|
||||
interest_score=interest_score,
|
||||
should_post_tweet=True
|
||||
)
|
||||
if not post_id:
|
||||
print(f"Failed to post to WordPress for '{title}'")
|
||||
logging.warning(f"Failed to post to WordPress for '{title}'")
|
||||
attempts += 1
|
||||
continue
|
||||
except Exception as e:
|
||||
print(f"WordPress posting error for '{title}': {e}")
|
||||
logging.error(f"Failed to post to WordPress for '{title}': {e}", exc_info=True)
|
||||
attempts += 1
|
||||
continue
|
||||
@@ -392,6 +417,7 @@ def curate_from_google_trends(geo_list=['US']):
|
||||
should_post_tweet=False
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Failed to update WordPress post '{title}' with share links: {e}")
|
||||
logging.error(f"Failed to update WordPress post '{title}' with share links: {e}", exc_info=True)
|
||||
finally:
|
||||
is_posting = False
|
||||
@@ -399,23 +425,29 @@ def curate_from_google_trends(geo_list=['US']):
|
||||
timestamp = datetime.now(timezone.utc).isoformat()
|
||||
save_json_file(POSTED_TITLES_FILE, title, timestamp)
|
||||
posted_titles.add(title)
|
||||
print(f"Successfully saved '{title}' to {POSTED_TITLES_FILE}")
|
||||
logging.info(f"Successfully saved '{title}' to {POSTED_TITLES_FILE}")
|
||||
|
||||
if image_url:
|
||||
save_json_file(USED_IMAGES_FILE, image_url, timestamp)
|
||||
used_images.add(image_url)
|
||||
print(f"Saved image '{image_url}' to {USED_IMAGES_FILE}")
|
||||
logging.info(f"Saved image '{image_url}' to {USED_IMAGES_FILE}")
|
||||
|
||||
print(f"***** SUCCESS: Posted '{post_data['title']}' (ID: {post_id}) from Google Trends *****")
|
||||
logging.info(f"***** SUCCESS: Posted '{post_data['title']}' (ID: {post_id}) from Google Trends *****")
|
||||
return post_data, category, True
|
||||
|
||||
attempts += 1
|
||||
print(f"WP posting failed for '{post_data['title']}'")
|
||||
logging.info(f"WP posting failed for '{post_data['title']}'")
|
||||
|
||||
print("No interesting Google Trend found after attempts")
|
||||
logging.info("No interesting Google Trend found after attempts")
|
||||
return None, None, False
|
||||
except Exception as e:
|
||||
logging.error(f"Unexpected error in curate_from_google_trends: {e}", exc_info=True)
|
||||
print(f"Unexpected error in curate_from_google_trends: {e}")
|
||||
return None, None, False
|
||||
|
||||
def run_google_trends_automator():
|
||||
|
||||
Reference in New Issue
Block a user