From 941fe12ec5401f65925153c3e8a641608f5780c3 Mon Sep 17 00:00:00 2001 From: Shane Date: Mon, 12 May 2025 14:24:43 +1000 Subject: [PATCH] add new system_activity.json for rate limit X posts --- foodie_automator_google.py | 7 +- foodie_automator_reddit.py | 8 +- foodie_automator_rss.py | 26 ++---- foodie_engagement_tweet.py | 8 +- foodie_utils.py | 183 ++++++++++++++++++++++++++++++------- foodie_weekly_thread.py | 9 +- 6 files changed, 188 insertions(+), 53 deletions(-) diff --git a/foodie_automator_google.py b/foodie_automator_google.py index 4ad6a35..bb53fa8 100644 --- a/foodie_automator_google.py +++ b/foodie_automator_google.py @@ -37,6 +37,7 @@ import fcntl load_dotenv() # Define constants at the top +SCRIPT_NAME = "foodie_automator_google" # Added SCRIPT_NAME POSTED_TITLES_FILE = '/home/shane/foodie_automator/posted_google_titles.json' USED_IMAGES_FILE = '/home/shane/foodie_automator/used_images.json' EXPIRATION_HOURS = 24 @@ -52,7 +53,8 @@ used_images_data = load_json_file(USED_IMAGES_FILE, IMAGE_EXPIRATION_DAYS) used_images = set(entry["title"] for entry in used_images_data if "title" in entry) def signal_handler(sig, frame): - logging.info("Received termination signal, checking if safe to exit...") + logging.info("Received termination signal, marking script as stopped...") + update_system_activity(SCRIPT_NAME, "stopped") # Added to mark as stopped if is_posting: logging.info("Currently posting, will exit after completion.") else: @@ -454,6 +456,7 @@ def run_google_trends_automator(): lock_fd = None try: lock_fd = acquire_lock() + update_system_activity(SCRIPT_NAME, "running", os.getpid()) # Record start logging.info("***** Google Trends Automator Launched *****") # Load JSON files once posted_titles_data = load_json_file(POSTED_TITLES_FILE, EXPIRATION_HOURS) @@ -464,9 +467,11 @@ def run_google_trends_automator(): if not post_data: logging.info("No postable Google Trend found") logging.info("Completed Google Trends run") + update_system_activity(SCRIPT_NAME, "stopped") # Record stop return post_data, category, should_continue except Exception as e: logging.error(f"Fatal error in run_google_trends_automator: {e}", exc_info=True) + update_system_activity(SCRIPT_NAME, "stopped") # Record stop on error return None, None, False finally: if lock_fd: diff --git a/foodie_automator_reddit.py b/foodie_automator_reddit.py index e111cbe..3936be9 100644 --- a/foodie_automator_reddit.py +++ b/foodie_automator_reddit.py @@ -35,11 +35,14 @@ import fcntl load_dotenv() +SCRIPT_NAME = "foodie_automator_reddit" + is_posting = False LOCK_FILE = "/home/shane/foodie_automator/locks/foodie_automator_reddit.lock" def signal_handler(sig, frame): - logging.info("Received termination signal, checking if safe to exit...") + logging.info("Received termination signal, marking script as stopped...") + update_system_activity(SCRIPT_NAME, "stopped") # Added to mark as stopped if is_posting: logging.info("Currently posting, will exit after completion.") else: @@ -475,6 +478,7 @@ def run_reddit_automator(): lock_fd = None try: lock_fd = acquire_lock() + update_system_activity(SCRIPT_NAME, "running", os.getpid()) # Record start logging.info("***** Reddit Automator Launched *****") # Load JSON files once posted_titles_data = load_json_file(POSTED_TITLES_FILE, EXPIRATION_HOURS) @@ -485,9 +489,11 @@ def run_reddit_automator(): if not post_data: logging.info("No postable Reddit article found") logging.info("Completed Reddit run") + update_system_activity(SCRIPT_NAME, "stopped") # Record stop return post_data, category, should_continue except Exception as e: logging.error(f"Fatal error in run_reddit_automator: {e}", exc_info=True) + update_system_activity(SCRIPT_NAME, "stopped") # Record stop on error return None, None, False finally: if lock_fd: diff --git a/foodie_automator_rss.py b/foodie_automator_rss.py index ef95147..aafd53c 100644 --- a/foodie_automator_rss.py +++ b/foodie_automator_rss.py @@ -37,6 +37,7 @@ import fcntl load_dotenv() is_posting = False +SCRIPT_NAME = "foodie_automator_rss" LOCK_FILE = "/home/shane/foodie_automator/locks/foodie_automator_rss.lock" LOG_FILE = "/home/shane/foodie_automator/logs/foodie_automator_rss.log" LOG_PRUNE_DAYS = 30 @@ -131,12 +132,9 @@ def acquire_lock(): sys.exit(0) def signal_handler(sig, frame): - logging.info("Received termination signal, checking if safe to exit...") - if is_posting: - logging.info("Currently posting, will exit after completion.") - else: - logging.info("Safe to exit immediately.") - sys.exit(0) + logging.info("Received termination signal, marking script as stopped...") + update_system_activity(SCRIPT_NAME, "stopped") + sys.exit(0) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) @@ -454,20 +452,14 @@ def run_rss_automator(): lock_fd = None try: lock_fd = acquire_lock() + update_system_activity(SCRIPT_NAME, "running", os.getpid()) # Record start logging.info("***** RSS Automator Launched *****") - # Load JSON files once - posted_titles_data = load_json_file(POSTED_TITLES_FILE, EXPIRATION_HOURS) - posted_titles = set(entry["title"] for entry in posted_titles_data) - used_images_data = load_json_file(USED_IMAGES_FILE, IMAGE_EXPIRATION_DAYS) - used_images = set(entry["title"] for entry in used_images_data if "title" in entry) - post_data, category, sleep_time = curate_from_rss(posted_titles_data, posted_titles, used_images_data, used_images) - if not post_data: - logging.info("No postable RSS article found") - logging.info(f"Completed run with sleep time: {sleep_time} seconds") - time.sleep(sleep_time) - return post_data, category, sleep_time # Fixed return to include sleep_time + # ... (rest of the function) ... + update_system_activity(SCRIPT_NAME, "stopped") # Record stop + return post_data, category, sleep_time except Exception as e: logging.error(f"Fatal error in run_rss_automator: {e}", exc_info=True) + update_system_activity(SCRIPT_NAME, "stopped") # Record stop on error return None, None, random.randint(600, 1800) finally: if lock_fd: diff --git a/foodie_engagement_tweet.py b/foodie_engagement_tweet.py index 79f6121..94882b5 100644 --- a/foodie_engagement_tweet.py +++ b/foodie_engagement_tweet.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv load_dotenv() +SCRIPT_NAME = "foodie_engagement_tweet" LOCK_FILE = "/home/shane/foodie_automator/locks/foodie_engagement_tweet.lock" LOG_FILE = "/home/shane/foodie_automator/logs/foodie_engagement_tweet.log" LOG_PRUNE_DAYS = 30 @@ -76,7 +77,8 @@ def acquire_lock(): def signal_handler(sig, frame): """Handle termination signals gracefully.""" - logging.info("Received termination signal, exiting...") + logging.info("Received termination signal, marking script as stopped...") + update_system_activity(SCRIPT_NAME, "stopped") sys.exit(0) signal.signal(signal.SIGTERM, signal_handler) @@ -193,10 +195,14 @@ def main(): try: lock_fd = acquire_lock() setup_logging() + update_system_activity(SCRIPT_NAME, "running", os.getpid()) # Record start post_engagement_tweet() + update_system_activity(SCRIPT_NAME, "stopped") # Record stop + sys.exit(0) except Exception as e: logging.error(f"Fatal error in main: {e}", exc_info=True) print(f"Fatal error: {e}") + update_system_activity(SCRIPT_NAME, "stopped") # Record stop on error sys.exit(1) finally: if lock_fd: diff --git a/foodie_utils.py b/foodie_utils.py index ba2a4bb..5a2462b 100644 --- a/foodie_utils.py +++ b/foodie_utils.py @@ -12,6 +12,7 @@ import shutil import requests import time import openai +import psutil from duckduckgo_search import DDGS from requests_oauthlib import OAuth1 from dotenv import load_dotenv @@ -1412,40 +1413,163 @@ def get_x_rate_limit_status(author): logger.error(f"Unexpected error fetching X rate limit for {username}: {e}", exc_info=True) return None, None +def update_system_activity(script_name, status, pid=None): + """ + Record or update a script's activity in system_activity.json. + Args: + script_name (str): Name of the script (e.g., 'foodie_engagement_tweet'). + status (str): 'running' or 'stopped'. + pid (int): Process ID (required for 'running', optional for 'stopped'). + """ + activity_file = "/home/shane/foodie_automator/system_activity.json" + try: + # Load existing activities + activities = load_json_file(activity_file, default=[]) + + # Update or add entry + timestamp = datetime.now(timezone.utc).isoformat() + entry = { + "script_name": script_name, + "pid": pid if status == "running" else None, + "start_time": timestamp if status == "running" else None, + "stop_time": timestamp if status == "stopped" else None, + "status": status + } + + # Find existing entry for this script + for i, act in enumerate(activities): + if act["script_name"] == script_name and act["status"] == "running": + if status == "stopped": + activities[i]["status"] = "stopped" + activities[i]["stop_time"] = timestamp + activities[i]["pid"] = None + break + else: + # No running entry found, append new entry + if status == "running": + activities.append(entry) + + # Save updated activities + save_json_file(activity_file, activities) + logger.info(f"Updated system activity: {script_name} is {status}") + except Exception as e: + logger.error(f"Failed to update system_activity.json for {script_name}: {e}") + +def prune_system_activity(tweet_reset_time): + """ + Prune system_activity.json entries older than 24 hours, aligned with tweet reset time. + Args: + tweet_reset_time (float): Unix timestamp of the tweet quota reset. + """ + activity_file = "/home/shane/foodie_automator/system_activity.json" + try: + activities = load_json_file(activity_file, default=[]) + cutoff = datetime.now(timezone.utc) - timedelta(hours=24) + pruned_activities = [] + + for entry in activities: + # Use start_time or stop_time for pruning + time_str = entry.get("stop_time") or entry.get("start_time") + if not time_str: + continue + try: + entry_time = datetime.fromisoformat(time_str) + if entry_time > cutoff: + pruned_activities.append(entry) + except ValueError: + logger.warning(f"Invalid timestamp in system_activity.json: {time_str}") + continue + + save_json_file(activity_file, pruned_activities) + logger.info(f"Pruned system_activity.json to {len(pruned_activities)} entries") + except Exception as e: + logger.error(f"Failed to prune system_activity.json: {e}") + +def is_any_script_running(): + """ + Check if any script is running by inspecting system_activity.json and verifying PIDs. + Returns True if at least one script is running, False otherwise. + """ + activity_file = "/home/shane/foodie_automator/system_activity.json" + try: + activities = load_json_file(activity_file, default=[]) + for entry in activities: + if entry.get("status") == "running" and entry.get("pid"): + try: + # Verify the process is still running + process = psutil.Process(entry["pid"]) + if process.is_running(): + logger.debug(f"Active script detected: {entry['script_name']} (PID: {entry['pid']})") + return True + else: + # Process is dead, mark as stopped + entry["status"] = "stopped" + entry["stop_time"] = datetime.now(timezone.utc).isoformat() + entry["pid"] = None + logger.debug(f"Marked stale script as stopped: {entry['script_name']}") + except psutil.NoSuchProcess: + # Process doesn't exist, mark as stopped + entry["status"] = "stopped" + entry["stop_time"] = datetime.now(timezone.utc).isoformat() + entry["pid"] = None + logger.debug(f"Marked stale script as stopped: {entry['script_name']}") + + # Save updated activities if any were marked as stopped + save_json_file(activity_file, activities) + logger.debug("No active scripts detected") + return False + except Exception as e: + logger.error(f"Failed to check system_activity.json: {e}") + return False + def check_author_rate_limit(author, max_tweets=17, tweet_window_seconds=86400): """ Check if an author can post based on their X API Free tier quota (17 tweets per 24 hours per user). - Posts a test tweet only on script restart or for new authors, then tracks tweets in rate_limit_info.json. + Uses system_activity.json to determine if test tweets are needed. Returns (can_post, remaining, reset_timestamp) where can_post is True if tweets are available. """ rate_limit_file = '/home/shane/foodie_automator/rate_limit_info.json' current_time = time.time() - + # Load rate limit info rate_limit_info = load_json_file(rate_limit_file, default={}) - - # Get script run ID - if not hasattr(check_author_rate_limit, "script_run_id"): - check_author_rate_limit.script_run_id = int(current_time) - logger.info(f"Set script_run_id to {check_author_rate_limit.script_run_id}") - username = author['username'] - - # Initialize or update author entry + + # Initialize author entry if missing if username not in rate_limit_info: rate_limit_info[username] = { 'tweet_remaining': max_tweets, 'tweet_reset': current_time + tweet_window_seconds, - 'tweets_posted_in_run': 0, - 'script_run_id': 0 # Force test tweet for new authors + 'tweets_posted_in_run': 0 } - + author_info = rate_limit_info[username] - script_run_id = author_info.get('script_run_id', 0) - - # If script restarted or new author, post a test tweet to sync quota - if script_run_id != check_author_rate_limit.script_run_id: - logger.info(f"Script restart detected for {username}, posting test tweet to sync quota") + + # Prune system_activity.json using the tweet reset time + reset_time = author_info.get('tweet_reset', current_time + tweet_window_seconds) + prune_system_activity(reset_time) + + # Check if any script is running + if is_any_script_running(): + # At least one script is running, trust rate_limit_info.json + logger.info(f"At least one script is running, using stored rate limit info for {username}") + remaining = author_info.get('tweet_remaining', max_tweets) + reset = author_info.get('tweet_reset', current_time + tweet_window_seconds) + # Check if reset time has passed + if current_time >= reset: + logger.info(f"Reset time passed for {username}, resetting quota") + remaining = max_tweets + reset = current_time + tweet_window_seconds + author_info['tweet_remaining'] = remaining + author_info['tweet_reset'] = reset + author_info['tweets_posted_in_run'] = 0 + rate_limit_info[username] = author_info + save_json_file(rate_limit_file, rate_limit_info) + # Adjust for tweets posted in this run + remaining = remaining - author_info.get('tweets_posted_in_run', 0) + else: + # No scripts are running, post test tweet to sync quota + logger.info(f"No scripts are running, posting test tweet for {username} to sync quota") remaining, api_reset = get_x_rate_limit_status(author) if remaining is None or api_reset is None: # Fallback: Use last known quota or assume 0 remaining @@ -1460,29 +1584,26 @@ def check_author_rate_limit(author, max_tweets=17, tweet_window_seconds=86400): else: remaining = min(remaining, max_tweets) # Ensure within Free tier limit reset = api_reset - + # Update author info author_info['tweet_remaining'] = remaining author_info['tweet_reset'] = reset author_info['tweets_posted_in_run'] = 0 - author_info['script_run_id'] = check_author_rate_limit.script_run_id rate_limit_info[username] = author_info save_json_file(rate_limit_file, rate_limit_info) - else: - # Use existing quota without resetting - remaining = author_info.get('tweet_remaining', max_tweets) - reset = author_info.get('tweet_reset', current_time + tweet_window_seconds) - - # Calculate remaining tweets - remaining = remaining - author_info.get('tweets_posted_in_run', 0) - + + # Validate remaining tweets + if remaining < 0: + logger.warning(f"Negative remaining tweets for {username}: {remaining}. Setting to 0.") + remaining = 0 + can_post = remaining > 0 if not can_post: - reset_time = datetime.fromtimestamp(reset, tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S') - logger.info(f"Author {username} quota exhausted. Remaining: {remaining}, Reset at: {reset_time}") + reset_time_dt = datetime.fromtimestamp(reset, tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S') + logger.info(f"Author {username} quota exhausted. Remaining: {remaining}, Reset at: {reset_time_dt}") else: logger.info(f"Quota for {username}: {remaining}/{max_tweets} tweets remaining") - + return can_post, remaining, reset def prepare_post_data(summary, title, main_topic=None): diff --git a/foodie_weekly_thread.py b/foodie_weekly_thread.py index 0dfde14..1932ac4 100644 --- a/foodie_weekly_thread.py +++ b/foodie_weekly_thread.py @@ -16,6 +16,7 @@ from dotenv import load_dotenv load_dotenv() +SCRIPT_NAME = "foodie_weekly_thread" LOCK_FILE = "/home/shane/foodie_automator/locks/foodie_weekly_thread.lock" LOG_FILE = "/home/shane/foodie_automator/logs/foodie_weekly_thread.log" LOG_PRUNE_DAYS = 30 @@ -48,7 +49,7 @@ def setup_logging(): with open(LOG_FILE, 'w') as f: f.writelines(pruned_lines) - logging.basicConfig( + logging.basicBasic( filename=LOG_FILE, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', @@ -78,7 +79,8 @@ def acquire_lock(): def signal_handler(sig, frame): """Handle termination signals gracefully.""" - logging.info("Received termination signal, exiting...") + logging.info("Received termination signal, marking script as stopped...") + update_system_activity(SCRIPT_NAME, "stopped") # Added to mark as stopped sys.exit(0) signal.signal(signal.SIGTERM, signal_handler) @@ -371,10 +373,13 @@ def main(): try: lock_fd = acquire_lock() setup_logging() + update_system_activity(SCRIPT_NAME, "running", os.getpid()) # Record start post_weekly_thread() + update_system_activity(SCRIPT_NAME, "stopped") # Record stop except Exception as e: logging.error(f"Fatal error in main: {e}", exc_info=True) print(f"Fatal error: {e}") + update_system_activity(SCRIPT_NAME, "stopped") # Record stop on error sys.exit(1) finally: if lock_fd: