add check once for rate limiting X
This commit is contained in:
+95
-62
@@ -162,23 +162,23 @@ def generate_article_tweet(author, post, persona):
|
||||
logging.info(f"Generated tweet: {tweet}")
|
||||
return tweet
|
||||
|
||||
def post_tweet(author, content, media_ids=None, reply_to_id=None):
|
||||
def post_tweet(author, content, media_ids=None, reply_to_id=None, tweet_type="rss"):
|
||||
"""
|
||||
Post a tweet for an author using X API v2.
|
||||
Returns (tweet_id, tweet_data) if successful, (None, None) if rate-limited or failed.
|
||||
Post a tweet for the given author using X API v2.
|
||||
Returns (tweet_id, tweet_data) on success, (None, None) on failure.
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
username = author['username']
|
||||
credentials = X_API_CREDENTIALS.get(username)
|
||||
if not credentials:
|
||||
logger.error(f"No X API credentials for {username}")
|
||||
logger.error(f"No X API credentials found for {username}")
|
||||
return None, None
|
||||
|
||||
# Check rate limit
|
||||
can_post, remaining, reset = check_author_rate_limit(author)
|
||||
if not can_post:
|
||||
reset_time = datetime.fromtimestamp(reset, tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
|
||||
logger.info(f"Cannot post tweet for {username}: rate-limited. Remaining: {remaining}, Reset at: {reset_time}")
|
||||
logger.info(f"Cannot post {tweet_type} tweet for {username}: rate-limited. Remaining: {remaining}, Reset at: {reset_time}")
|
||||
return None, None
|
||||
|
||||
oauth = OAuth1(
|
||||
@@ -198,31 +198,37 @@ def post_tweet(author, content, media_ids=None, reply_to_id=None):
|
||||
response = requests.post(url, json=payload, auth=oauth)
|
||||
headers = response.headers
|
||||
|
||||
# Update rate limit info
|
||||
# Update in-run tweet counter
|
||||
rate_limit_file = '/home/shane/foodie_automator/rate_limit_info.json'
|
||||
rate_limit_info = load_json_file(rate_limit_file, default={})
|
||||
remaining = int(headers.get('x-user-limit-24hour-remaining', remaining))
|
||||
reset = int(headers.get('x-user-limit-24hour-reset', reset))
|
||||
rate_limit_info[username] = {'tweet_remaining': remaining, 'tweet_reset': reset}
|
||||
save_json_file(rate_limit_file, rate_limit_info)
|
||||
if username in rate_limit_info:
|
||||
author_info = rate_limit_info[username]
|
||||
author_info['tweets_posted_in_run'] = author_info.get('tweets_posted_in_run', 0) + 1
|
||||
remaining = author_info['tweet_remaining'] - author_info['tweets_posted_in_run']
|
||||
rate_limit_info[username] = author_info
|
||||
save_json_file(rate_limit_file, rate_limit_info)
|
||||
logger.info(f"Updated in-run tweet counter for {username} ({tweet_type}): {remaining}/17 tweets remaining")
|
||||
else:
|
||||
logger.warning(f"Rate limit info not found for {username}, assuming quota exhausted")
|
||||
remaining = 0
|
||||
|
||||
if response.status_code == 201:
|
||||
tweet_data = response.json()
|
||||
tweet_id = tweet_data.get('data', {}).get('id')
|
||||
logger.info(f"Successfully tweeted for {username}: {content[:50]}... (ID: {tweet_id})")
|
||||
logger.info(f"Successfully tweeted {tweet_type} for {username}: {content[:50]}... (ID: {tweet_id})")
|
||||
return tweet_id, tweet_data
|
||||
elif response.status_code == 429:
|
||||
logger.info(f"Rate limit exceeded for {username}: {remaining} remaining, reset at {datetime.fromtimestamp(reset, tz=timezone.utc)}")
|
||||
logger.info(f"Rate limit exceeded for {username} ({tweet_type}): {remaining} remaining, reset at {datetime.fromtimestamp(reset, tz=timezone.utc)}")
|
||||
return None, None
|
||||
elif response.status_code == 403:
|
||||
logger.error(f"403 Forbidden for {username}: {response.text}")
|
||||
logger.error(f"403 Forbidden for {username} ({tweet_type}): {response.text}")
|
||||
return None, None
|
||||
else:
|
||||
logger.error(f"Failed to tweet for {username}: {response.status_code} - {response.text}")
|
||||
logger.error(f"Failed to post {tweet_type} tweet for {username}: {response.status_code} - {response.text}")
|
||||
return None, None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error posting tweet for {username}: {e}", exc_info=True)
|
||||
logger.error(f"Unexpected error posting {tweet_type} tweet for {username}: {e}", exc_info=True)
|
||||
return None, None
|
||||
|
||||
def select_best_persona(interest_score, content=""):
|
||||
@@ -832,7 +838,7 @@ def post_to_wp(post_data, category, link, author, image_url, original_source, im
|
||||
timestamp = datetime.now(timezone.utc).isoformat()
|
||||
save_post_to_recent(post_data["title"], post_url, wp_username, timestamp)
|
||||
|
||||
# Post tweet if enabled
|
||||
# Post tweet if enabled
|
||||
if should_post_tweet:
|
||||
credentials = X_API_CREDENTIALS.get(post_data["author"])
|
||||
if credentials:
|
||||
@@ -845,7 +851,7 @@ def post_to_wp(post_data, category, link, author, image_url, original_source, im
|
||||
"url": post_url
|
||||
}
|
||||
tweet_text = generate_article_tweet(author, tweet_post, persona)
|
||||
tweet_id, tweet_data = post_tweet(author, tweet_text)
|
||||
tweet_id, tweet_data = post_tweet(author, tweet_text, tweet_type="rss")
|
||||
if tweet_id:
|
||||
logger.info(f"Successfully tweeted for post: {post_data['title']} (Tweet ID: {tweet_id})")
|
||||
else:
|
||||
@@ -1162,52 +1168,81 @@ def select_best_author(content, interest_score):
|
||||
|
||||
def check_author_rate_limit(author, max_tweets=17, tweet_window_seconds=86400):
|
||||
"""
|
||||
Check if an author is rate-limited for tweets using real-time X API v2 data.
|
||||
Check if an author can post based on their X API Free tier quota (17 tweets per 24 hours per app).
|
||||
Posts a test tweet only on script restart or for new authors, then tracks tweets in rate_limit_info.json.
|
||||
Returns (can_post, remaining, reset_timestamp) where can_post is True if tweets are available.
|
||||
Caches API results in memory for 5 minutes.
|
||||
Falls back to rate_limit_info.json or assumes 1 tweet remaining if API fails.
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
rate_limit_file = '/home/shane/foodie_automator/rate_limit_info.json'
|
||||
current_time = time.time()
|
||||
|
||||
# In-memory cache
|
||||
if not hasattr(check_author_rate_limit, "cache"):
|
||||
check_author_rate_limit.cache = {}
|
||||
# Load rate limit info
|
||||
rate_limit_info = load_json_file(rate_limit_file, default={})
|
||||
|
||||
# Get script run ID (set at startup in foodie_automator_rss.py)
|
||||
if not hasattr(check_author_rate_limit, "script_run_id"):
|
||||
check_author_rate_limit.script_run_id = int(current_time)
|
||||
logger.info(f"Set script_run_id to {check_author_rate_limit.script_run_id}")
|
||||
|
||||
username = author['username']
|
||||
cache_key = f"{username}_{int(current_time // 300)}" # Cache for 5 minutes
|
||||
|
||||
if cache_key in check_author_rate_limit.cache:
|
||||
remaining, reset = check_author_rate_limit.cache[cache_key]
|
||||
logger.debug(f"Using cached rate limit for {username}: {remaining} remaining, reset at {datetime.fromtimestamp(reset, tz=timezone.utc)}")
|
||||
else:
|
||||
# Initialize or update author entry
|
||||
if username not in rate_limit_info:
|
||||
rate_limit_info[username] = {
|
||||
'tweet_remaining': max_tweets,
|
||||
'tweet_reset': current_time + tweet_window_seconds,
|
||||
'tweets_posted_in_run': 0,
|
||||
'script_run_id': 0 # Force test tweet for new authors
|
||||
}
|
||||
|
||||
author_info = rate_limit_info[username]
|
||||
script_run_id = author_info.get('script_run_id', 0)
|
||||
|
||||
# If script restarted or new author, post a test tweet to sync quota
|
||||
if script_run_id != check_author_rate_limit.script_run_id:
|
||||
logger.info(f"Script restart detected for {username}, posting test tweet to sync quota")
|
||||
remaining, reset = get_x_rate_limit_status(author)
|
||||
if remaining is None or reset is None:
|
||||
# Fallback: Load from rate_limit_info.json or assume 1 tweet remaining
|
||||
rate_limit_info = load_json_file(rate_limit_file, default={})
|
||||
if username not in rate_limit_info or current_time >= rate_limit_info.get(username, {}).get('tweet_reset', 0):
|
||||
rate_limit_info[username] = {
|
||||
'tweet_remaining': 1, # Allow one tweet to avoid blocking
|
||||
'tweet_reset': current_time + tweet_window_seconds
|
||||
}
|
||||
save_json_file(rate_limit_file, rate_limit_info)
|
||||
remaining = rate_limit_info[username].get('tweet_remaining', 1)
|
||||
reset = rate_limit_info[username].get('tweet_reset', current_time + tweet_window_seconds)
|
||||
logger.warning(f"X API rate limit check failed for {username}, using fallback: {remaining} remaining")
|
||||
check_author_rate_limit.cache[cache_key] = (remaining, reset)
|
||||
# Fallback: Use last known quota or assume 0 remaining
|
||||
if current_time < author_info.get('tweet_reset', 0):
|
||||
remaining = author_info.get('tweet_remaining', 0)
|
||||
reset = author_info.get('tweet_reset', current_time + tweet_window_seconds)
|
||||
logger.warning(f"Test tweet failed for {username}, using last known quota: {remaining} remaining")
|
||||
else:
|
||||
remaining = max_tweets
|
||||
reset = current_time + tweet_window_seconds
|
||||
logger.warning(f"Test tweet failed for {username}, resetting quota to {max_tweets}")
|
||||
# Update author info with synced quota
|
||||
author_info = {
|
||||
'tweet_remaining': remaining,
|
||||
'tweet_reset': reset,
|
||||
'tweets_posted_in_run': 0,
|
||||
'script_run_id': check_author_rate_limit.script_run_id
|
||||
}
|
||||
rate_limit_info[username] = author_info
|
||||
save_json_file(rate_limit_file, rate_limit_info)
|
||||
|
||||
# Calculate remaining tweets based on tweets posted in this run
|
||||
remaining = author_info['tweet_remaining'] - author_info['tweets_posted_in_run']
|
||||
reset = author_info['tweet_reset']
|
||||
|
||||
# Check if quota has reset
|
||||
if current_time >= reset:
|
||||
logger.info(f"Quota reset for {username}, restoring to {max_tweets} tweets")
|
||||
remaining = max_tweets
|
||||
reset = current_time + tweet_window_seconds
|
||||
author_info['tweet_remaining'] = remaining
|
||||
author_info['tweet_reset'] = reset
|
||||
author_info['tweets_posted_in_run'] = 0
|
||||
rate_limit_info[username] = author_info
|
||||
save_json_file(rate_limit_file, rate_limit_info)
|
||||
|
||||
can_post = remaining > 0
|
||||
if not can_post:
|
||||
reset_time = datetime.fromtimestamp(reset, tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
|
||||
logger.info(f"Author {username} is rate-limited. Remaining: {remaining}, Reset at: {reset_time}")
|
||||
logger.info(f"Author {username} quota exhausted. Remaining: {remaining}, Reset at: {reset_time}")
|
||||
else:
|
||||
logger.info(f"Rate limit for {username}: {remaining}/{max_tweets} tweets remaining")
|
||||
|
||||
# Update rate_limit_info.json
|
||||
rate_limit_info = load_json_file(rate_limit_file, default={})
|
||||
rate_limit_info[username] = {'tweet_remaining': remaining, 'tweet_reset': reset}
|
||||
save_json_file(rate_limit_file, rate_limit_info)
|
||||
logger.info(f"Quota for {username}: {remaining}/{max_tweets} tweets remaining")
|
||||
|
||||
return can_post, remaining, reset
|
||||
|
||||
@@ -1245,9 +1280,8 @@ def get_next_author_round_robin():
|
||||
|
||||
def get_x_rate_limit_status(author):
|
||||
"""
|
||||
Check the X API v2 rate limit status for an author by attempting a test tweet.
|
||||
Returns (remaining, reset) where remaining is the number of tweets left in the 24-hour window,
|
||||
and reset is the Unix timestamp when the limit resets.
|
||||
Check the X API Free tier rate limit by posting a test tweet.
|
||||
Returns (remaining, reset) based on app-level headers (x-rate-limit-remaining, x-rate-limit-reset).
|
||||
Returns (None, None) if the check fails.
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -1265,17 +1299,17 @@ def get_x_rate_limit_status(author):
|
||||
)
|
||||
url = 'https://api.x.com/2/tweets'
|
||||
payload = {'text': f'Test tweet to check rate limits for {username} - please ignore {int(time.time())}'}
|
||||
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, auth=oauth)
|
||||
headers = response.headers
|
||||
|
||||
# Extract rate limit info from headers
|
||||
remaining = int(headers.get('x-user-limit-24hour-remaining', 0))
|
||||
reset = int(headers.get('x-user-limit-24hour-reset', 0))
|
||||
|
||||
# Extract app-level rate limit info from headers
|
||||
remaining = int(headers.get('x-rate-limit-remaining', 0))
|
||||
reset = int(headers.get('x-rate-limit-reset', 0))
|
||||
|
||||
if response.status_code == 201:
|
||||
# Tweet posted successfully, delete it
|
||||
# Delete the test tweet
|
||||
tweet_id = response.json().get('data', {}).get('id')
|
||||
if tweet_id:
|
||||
delete_url = f'https://api.x.com/2/tweets/{tweet_id}'
|
||||
@@ -1283,20 +1317,19 @@ def get_x_rate_limit_status(author):
|
||||
if delete_response.status_code == 200:
|
||||
logger.info(f"Successfully deleted test tweet {tweet_id} for {username}")
|
||||
else:
|
||||
logger.warning(f"Failed to delete test tweet {tweet_id} for {username}: {delete_response.status_code}")
|
||||
logger.warning(f"Failed to delete test tweet {tweet_id} for {username}: {delete_response.status_code} - {delete_response.text}")
|
||||
logger.info(f"Rate limit for {username}: {remaining} remaining, reset at {datetime.fromtimestamp(reset, tz=timezone.utc)}")
|
||||
return remaining, reset
|
||||
elif response.status_code == 429:
|
||||
# Rate limit exceeded
|
||||
logger.info(f"Rate limit exceeded for {username}: {remaining} remaining, reset at {datetime.fromtimestamp(reset, tz=timezone.utc)}")
|
||||
return remaining, reset
|
||||
elif response.status_code == 403:
|
||||
# Forbidden (e.g., account restrictions), but headers may still provide rate limit info
|
||||
logger.warning(f"403 Forbidden for {username}: {response.text}, rate limit info: {remaining} remaining, reset at {datetime.fromtimestamp(reset, tz=timezone.utc)}")
|
||||
return remaining, reset
|
||||
else:
|
||||
logger.error(f"Unexpected response for {username}: {response.status_code} - {response.text}")
|
||||
return None, None
|
||||
|
||||
logger.info(f"Rate limit for {username}: {remaining} remaining, reset at {datetime.fromtimestamp(reset, tz=timezone.utc)}")
|
||||
return remaining, reset
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error fetching X rate limit for {username}: {e}", exc_info=True)
|
||||
return None, None
|
||||
|
||||
Reference in New Issue
Block a user