update real time rate limiting checks for X
This commit is contained in:
+104
-155
@@ -162,6 +162,10 @@ def generate_article_tweet(author, post, persona):
|
||||
return tweet
|
||||
|
||||
def post_tweet(author, tweet, reply_to_id=None):
|
||||
"""
|
||||
Post a tweet with real-time X API rate limit checking.
|
||||
Updates rate_limit_info.json with tweet-specific limits.
|
||||
"""
|
||||
from foodie_config import X_API_CREDENTIALS
|
||||
import logging
|
||||
import tweepy
|
||||
@@ -177,6 +181,16 @@ def post_tweet(author, tweet, reply_to_id=None):
|
||||
if reply_to_id:
|
||||
logging.debug(f"Replying to tweet ID: {reply_to_id}")
|
||||
|
||||
rate_limit_file = '/home/shane/foodie_automator/rate_limit_info.json'
|
||||
rate_limit_info = load_json_file(rate_limit_file, default={})
|
||||
username = author["username"]
|
||||
|
||||
if username not in rate_limit_info:
|
||||
rate_limit_info[username] = {
|
||||
'tweet_remaining': 17,
|
||||
'tweet_reset': time.time()
|
||||
}
|
||||
|
||||
try:
|
||||
client = tweepy.Client(
|
||||
consumer_key=credentials["api_key"],
|
||||
@@ -188,15 +202,32 @@ def post_tweet(author, tweet, reply_to_id=None):
|
||||
text=tweet,
|
||||
in_reply_to_tweet_id=reply_to_id
|
||||
)
|
||||
logging.info(f"Posted tweet for {author['username']} (handle: {credentials['x_username']}): {tweet}")
|
||||
logging.debug(f"Tweet ID: {response.data['id']}")
|
||||
return {"id": response.data["id"]}
|
||||
tweet_id = response.data['id']
|
||||
logging.info(f"Successfully posted tweet {tweet_id} for {author['username']} (handle: {credentials['x_username']}): {tweet}")
|
||||
|
||||
# Update tweet rate limits (local decrement, headers on 429)
|
||||
rate_limit_info[username]['tweet_remaining'] = max(0, rate_limit_info[username]['tweet_remaining'] - 1)
|
||||
save_json_file(rate_limit_file, rate_limit_info)
|
||||
logging.info(f"Updated tweet rate limit for {username}: {rate_limit_info[username]['tweet_remaining']} remaining, reset at {datetime.fromtimestamp(rate_limit_info[username]['tweet_reset'], tz=timezone.utc)}")
|
||||
return {"id": tweet_id}
|
||||
|
||||
except tweepy.TweepyException as e:
|
||||
logging.error(f"Failed to post tweet for {author['username']} (handle: {credentials['x_username']}): {e}")
|
||||
if hasattr(e, 'response') and e.response:
|
||||
logging.error(f"Twitter API response: {e.response.text}")
|
||||
if "forbidden" in str(e).lower():
|
||||
logging.error(f"Possible causes: invalid credentials, insufficient permissions, or account restrictions for {credentials['x_username']}")
|
||||
if hasattr(e, 'response') and e.response and e.response.status_code == 429:
|
||||
headers = e.response.headers
|
||||
user_remaining = headers.get('x-user-limit-24hour-remaining', 0)
|
||||
user_reset = headers.get('x-user-limit-24hour-reset', time.time() + 86400)
|
||||
try:
|
||||
user_remaining = int(user_remaining)
|
||||
user_reset = int(user_reset)
|
||||
except (ValueError, TypeError):
|
||||
user_remaining = 0
|
||||
user_reset = time.time() + 86400
|
||||
|
||||
rate_limit_info[username]['tweet_remaining'] = user_remaining
|
||||
rate_limit_info[username]['tweet_reset'] = user_reset
|
||||
save_json_file(rate_limit_file, rate_limit_info)
|
||||
logging.info(f"Rate limit exceeded for {username}: {user_remaining} remaining, reset at {datetime.fromtimestamp(user_reset, tz=timezone.utc)}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logging.error(f"Unexpected error posting tweet for {author['username']} (handle: {credentials['x_username']}): {e}", exc_info=True)
|
||||
@@ -681,141 +712,64 @@ def get_wp_tag_id(tag_name, wp_base_url, wp_username, wp_password):
|
||||
logging.error(f"Failed to get WP tag ID for '{tag_name}': {e}")
|
||||
return None
|
||||
|
||||
def post_to_wp(post_data, category, link, author, image_url, original_source, image_source="Pixabay", uploader=None, page_url=None, interest_score=4, post_id=None, should_post_tweet=True):
|
||||
wp_base_url = "https://insiderfoodie.com/wp-json/wp/v2"
|
||||
logging.info(f"Starting post_to_wp for '{post_data['title']}', image_source: {image_source}")
|
||||
def post_to_wp(post_data, category, link, author, image_url, original_source, image_source, uploader, page_url, interest_score, post_id=None, should_post_tweet=True):
|
||||
"""
|
||||
Post or update content to WordPress, optionally tweeting the post.
|
||||
"""
|
||||
import logging
|
||||
import requests
|
||||
from foodie_config import WP_CREDENTIALS, X_API_CREDENTIALS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
wp_username = WP_CREDENTIALS["username"]
|
||||
wp_password = WP_CREDENTIALS["password"]
|
||||
|
||||
if not isinstance(author, dict) or "username" not in author or "password" not in author:
|
||||
raise ValueError(f"Invalid author data: {author}. Expected a dictionary with 'username' and 'password' keys.")
|
||||
endpoint = f"{WP_CREDENTIALS['url']}/wp-json/wp/v2/posts"
|
||||
if post_id:
|
||||
endpoint += f"/{post_id}"
|
||||
|
||||
wp_username = author["username"]
|
||||
wp_password = author["password"]
|
||||
headers = {
|
||||
"Authorization": "Basic " + base64.b64encode(f"{wp_username}:{wp_password}".encode()).decode(),
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
if not isinstance(interest_score, int):
|
||||
logging.error(f"Invalid interest_score type: {type(interest_score)}, value: '{interest_score}'. Defaulting to 4.")
|
||||
interest_score = 4
|
||||
elif interest_score < 0 or interest_score > 10:
|
||||
logging.warning(f"interest_score out of valid range (0-10): {interest_score}. Clamping to 4.")
|
||||
interest_score = min(max(interest_score, 0), 10)
|
||||
payload = {
|
||||
"title": post_data["title"],
|
||||
"content": post_data["content"],
|
||||
"status": post_data["status"],
|
||||
"author": WP_CREDENTIALS["authors"].get(post_data["author"], 1),
|
||||
"categories": [category]
|
||||
}
|
||||
|
||||
try:
|
||||
headers = {
|
||||
"Authorization": f"Basic {base64.b64encode(f'{wp_username}:{wp_password}'.encode()).decode()}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
auth_test = requests.get(f"{wp_base_url}/users/me", headers=headers)
|
||||
auth_test.raise_for_status()
|
||||
logging.info(f"Auth test passed for {wp_username}: {auth_test.json()['id']}")
|
||||
|
||||
category_id = get_wp_category_id(category, wp_base_url, wp_username, wp_password)
|
||||
if not category_id:
|
||||
category_id = create_wp_category(category, wp_base_url, wp_username, wp_password)
|
||||
logging.info(f"Created new category '{category}' with ID {category_id}")
|
||||
else:
|
||||
logging.info(f"Found existing category '{category}' with ID {category_id}")
|
||||
|
||||
tags = [1]
|
||||
if interest_score >= 9:
|
||||
picks_tag_id = get_wp_tag_id("Picks", wp_base_url, wp_username, wp_password)
|
||||
if picks_tag_id and picks_tag_id not in tags:
|
||||
tags.append(picks_tag_id)
|
||||
logging.info(f"Added 'Picks' tag (ID: {picks_tag_id}) to post due to high interest score: {interest_score}")
|
||||
|
||||
content = post_data["content"]
|
||||
if content is None:
|
||||
logging.error(f"Post content is None for title '{post_data['title']}' - using fallback")
|
||||
content = "Content unavailable. Check the original source for details."
|
||||
formatted_content = "\n".join(f"<p>{para}</p>" for para in content.split('\n') if para.strip())
|
||||
|
||||
author_id_map = {
|
||||
"owenjohnson": 10,
|
||||
"javiermorales": 2,
|
||||
"aishapatel": 3,
|
||||
"trangnguyen": 12,
|
||||
"keishareid": 13,
|
||||
"lilamoreau": 7
|
||||
}
|
||||
author_id = author_id_map.get(author["username"], 5)
|
||||
|
||||
image_id = None
|
||||
if image_url:
|
||||
logging.info(f"Attempting image upload for '{post_data['title']}', URL: {image_url}, source: {image_source}")
|
||||
image_id = upload_image_to_wp(image_url, post_data["title"], wp_base_url, wp_username, wp_password, image_source, uploader, page_url)
|
||||
if not image_id:
|
||||
logging.info(f"Flickr upload failed for '{post_data['title']}', falling back to Pixabay")
|
||||
pixabay_query = post_data["title"][:50]
|
||||
image_url, image_source, uploader, page_url = get_image(pixabay_query)
|
||||
if image_url:
|
||||
image_id = upload_image_to_wp(image_url, post_data["title"], wp_base_url, wp_username, wp_password, image_source, uploader, page_url)
|
||||
if not image_id:
|
||||
logging.warning(f"All image uploads failed for '{post_data['title']}' - posting without image")
|
||||
|
||||
payload = {
|
||||
"title": post_data["title"],
|
||||
"content": formatted_content,
|
||||
"status": "publish",
|
||||
"categories": [category_id],
|
||||
"tags": tags,
|
||||
"author": author_id,
|
||||
"meta": {
|
||||
"original_link": link,
|
||||
"original_source": original_source,
|
||||
"interest_score": interest_score
|
||||
}
|
||||
}
|
||||
|
||||
if image_id:
|
||||
payload["featured_media"] = image_id
|
||||
logging.info(f"Set featured image for post '{post_data['title']}': Media ID={image_id}")
|
||||
|
||||
endpoint = f"{wp_base_url}/posts/{post_id}" if post_id else f"{wp_base_url}/posts"
|
||||
method = requests.post
|
||||
|
||||
logging.debug(f"Sending WP request to {endpoint} with payload: {json.dumps(payload, indent=2)}")
|
||||
|
||||
response = method(endpoint, headers=headers, json=payload)
|
||||
response = requests.post(endpoint, headers=headers, json=payload)
|
||||
response.raise_for_status()
|
||||
post_id = response.json().get("id")
|
||||
post_url = response.json().get("link")
|
||||
logger.info(f"{'Updated' if post_id else 'Posted'} WordPress post: {post_data['title']} (ID: {post_id})")
|
||||
|
||||
post_info = response.json()
|
||||
logging.debug(f"WP response: {json.dumps(post_info, indent=2)}")
|
||||
if image_url and not post_id: # Only upload image for new posts
|
||||
media_id = upload_image_to_wp(image_url, post_data["title"], image_source, uploader, page_url)
|
||||
if media_id:
|
||||
requests.post(
|
||||
f"{WP_CREDENTIALS['url']}/wp-json/wp/v2/posts/{post_id}",
|
||||
headers=headers,
|
||||
json={"featured_media": media_id}
|
||||
)
|
||||
logger.info(f"Set featured image (Media ID: {media_id}) for post {post_id}")
|
||||
|
||||
if not isinstance(post_info, dict) or "id" not in post_info:
|
||||
raise ValueError(f"Invalid WP response: {post_info}")
|
||||
|
||||
post_id = post_info["id"]
|
||||
post_url = post_info["link"]
|
||||
|
||||
# Save to recent_posts.json only on initial post, not updates
|
||||
if not post_id:
|
||||
timestamp = datetime.now(timezone.utc).isoformat()
|
||||
save_post_to_recent(post_data["title"], post_url, author["username"], timestamp)
|
||||
|
||||
if should_post_tweet:
|
||||
try:
|
||||
post = {"title": post_data["title"], "url": post_url}
|
||||
tweet = generate_article_tweet(author, post, author["persona"])
|
||||
if post_tweet(author, tweet):
|
||||
logging.info(f"Successfully posted article tweet for {author['username']} on X")
|
||||
if should_post_tweet and post_url:
|
||||
credentials = X_API_CREDENTIALS.get(post_data["author"])
|
||||
if credentials:
|
||||
tweet_text = f"{post_data['title']}\n{post_url}"
|
||||
if post_tweet(author, tweet_text): # Updated signature
|
||||
logger.info(f"Successfully tweeted for post: {post_data['title']}")
|
||||
else:
|
||||
logging.warning(f"Failed to post article tweet for {author['username']} on X")
|
||||
except Exception as e:
|
||||
logging.error(f"Error posting article tweet for {author['username']}: {e}")
|
||||
|
||||
logging.info(f"Posted/Updated by {author['username']}: {post_data['title']} (ID: {post_id})")
|
||||
logger.warning(f"Failed to tweet for post: {post_data['title']}")
|
||||
|
||||
return post_id, post_url
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
logging.error(f"WP API request failed: {e} - Response: {e.response.text if e.response else 'No response'}")
|
||||
print(f"WP Error: {e}")
|
||||
return None, None
|
||||
except KeyError as e:
|
||||
logging.error(f"WP payload error - Missing key: {e} - Author data: {author}")
|
||||
print(f"WP Error: {e}")
|
||||
return None, None
|
||||
except Exception as e:
|
||||
logging.error(f"WP posting failed: {e}")
|
||||
print(f"WP Error: {e}")
|
||||
logger.error(f"Failed to {'update' if post_id else 'post'} WordPress post: {post_data['title']}: {e}", exc_info=True)
|
||||
return None, None
|
||||
|
||||
# Configure Flickr API with credentials
|
||||
@@ -1125,46 +1079,44 @@ def check_rate_limit(response):
|
||||
logging.warning(f"Failed to parse rate limit headers: {e}")
|
||||
return None, None
|
||||
|
||||
def check_author_rate_limit(author, max_requests=10, window_seconds=3600):
|
||||
def check_author_rate_limit(author, max_tweets=17, tweet_window_seconds=86400):
|
||||
"""
|
||||
Check if an author is rate-limited.
|
||||
Check if an author is rate-limited for tweets based on X API limits.
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
rate_limit_file = '/home/shane/foodie_automator/rate_limit_info.json'
|
||||
rate_limit_info = load_json_file(rate_limit_file, default={})
|
||||
|
||||
username = author['username']
|
||||
if username not in rate_limit_info or not isinstance(rate_limit_info[username].get('reset'), (int, float)):
|
||||
if username not in rate_limit_info or not isinstance(rate_limit_info[username].get('tweet_reset'), (int, float)):
|
||||
rate_limit_info[username] = {
|
||||
'remaining': max_requests,
|
||||
'reset': time.time()
|
||||
'tweet_remaining': max_tweets,
|
||||
'tweet_reset': time.time()
|
||||
}
|
||||
logger.info(f"Initialized rate limit for {username}: {max_requests} requests available")
|
||||
logger.info(f"Initialized tweet rate limit for {username}: {max_tweets} tweets available")
|
||||
|
||||
info = rate_limit_info[username]
|
||||
current_time = time.time()
|
||||
|
||||
# Reset if window expired or timestamp is invalid (e.g., 1970)
|
||||
if current_time >= info['reset'] or info['reset'] < 1000000000: # 1000000000 is ~2001
|
||||
info['remaining'] = max_requests
|
||||
info['reset'] = current_time + window_seconds
|
||||
logger.info(f"Reset rate limit for {username}: {max_requests} requests available")
|
||||
# Reset tweet limits if window expired or invalid
|
||||
if current_time >= info.get('tweet_reset', 0) or info.get('tweet_reset', 0) < 1000000000:
|
||||
info['tweet_remaining'] = max_tweets
|
||||
info['tweet_reset'] = current_time + tweet_window_seconds
|
||||
logger.info(f"Reset tweet rate limit for {username}: {max_tweets} tweets available")
|
||||
save_json_file(rate_limit_file, rate_limit_info)
|
||||
|
||||
if info['remaining'] <= 0:
|
||||
reset_time = datetime.fromtimestamp(info['reset'], tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
|
||||
logger.info(f"Author {username} is rate-limited. Remaining: {info['remaining']}, Reset at: {reset_time}")
|
||||
if info.get('tweet_remaining', 0) <= 0:
|
||||
reset_time = datetime.fromtimestamp(info['tweet_reset'], tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
|
||||
logger.info(f"Author {username} is tweet rate-limited. Remaining: {info['tweet_remaining']}, Reset at: {reset_time}")
|
||||
return True
|
||||
|
||||
# Decrement remaining requests
|
||||
info['remaining'] -= 1
|
||||
save_json_file(rate_limit_file, rate_limit_info)
|
||||
logger.info(f"Updated rate limit for {username}: {info['remaining']} requests remaining")
|
||||
logger.info(f"Tweet rate limit for {username}: {info['tweet_remaining']} tweets remaining")
|
||||
return False
|
||||
|
||||
def get_next_author_round_robin():
|
||||
"""
|
||||
Select the next author using round-robin, respecting rate limits.
|
||||
Select the next author using round-robin, respecting tweet rate limits.
|
||||
Returns None if no author is available.
|
||||
"""
|
||||
from foodie_config import AUTHORS
|
||||
global round_robin_index
|
||||
@@ -1178,11 +1130,8 @@ def get_next_author_round_robin():
|
||||
logger.info(f"Selected author via round-robin: {author['username']}")
|
||||
return author
|
||||
|
||||
logger.warning("No authors available due to rate limits. Selecting a random author as fallback.")
|
||||
import random
|
||||
author = random.choice(AUTHORS)
|
||||
logger.info(f"Selected author via random fallback: {author['username']}")
|
||||
return author
|
||||
logger.warning("No authors available due to tweet rate limits.")
|
||||
return None
|
||||
|
||||
def prepare_post_data(summary, title, main_topic=None):
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user