foodie_dev
/
foodie-automator
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix image upload issue

Browse Source
main
Shane 7 months ago
parent 028dfc3fc8
commit 4368bf68a5
2 changed files with 75 additions and 63 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 112
      foodie_automator_rss.py
  2. 26
      foodie_utils.py

112
foodie_automator_rss.py
Unescape View File

@ -37,23 +37,12 @@ load_dotenv()
is_posting = False is_posting = False
LOCK_FILE = "/home/shane/foodie_automator/locks/foodie_automator_rss.lock" LOCK_FILE = "/home/shane/foodie_automator/locks/foodie_automator_rss.lock"
def signal_handler(sig, frame):
logging.info("Received termination signal, checking if safe to exit...")
if is_posting:
logging.info("Currently posting, will exit after completion.")
else:
logging.info("Safe to exit immediately.")
sys.exit(0)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
LOG_FILE = "/home/shane/foodie_automator/logs/foodie_automator_rss.log" LOG_FILE = "/home/shane/foodie_automator/logs/foodie_automator_rss.log"
LOG_PRUNE_DAYS = 30 LOG_PRUNE_DAYS = 30
FEED_TIMEOUT = 15 FEED_TIMEOUT = 15
MAX_RETRIES = 3 MAX_RETRIES = 3
RETRY_BACKOFF = 2 RETRY_BACKOFF = 2
IMAGE_UPLOAD_TIMEOUT = 30 # Added to match foodie_utils.py
POSTED_TITLES_FILE = '/home/shane/foodie_automator/posted_rss_titles.json' POSTED_TITLES_FILE = '/home/shane/foodie_automator/posted_rss_titles.json'
USED_IMAGES_FILE = '/home/shane/foodie_automator/used_images.json' USED_IMAGES_FILE = '/home/shane/foodie_automator/used_images.json'
@ -65,39 +54,48 @@ posted_titles = set(entry["title"] for entry in posted_titles_data)
used_images = set(entry["title"] for entry in load_json_file(USED_IMAGES_FILE, IMAGE_EXPIRATION_DAYS) if "title" in entry) used_images = set(entry["title"] for entry in load_json_file(USED_IMAGES_FILE, IMAGE_EXPIRATION_DAYS) if "title" in entry)
def setup_logging(): def setup_logging():
if os.path.exists(LOG_FILE): """Initialize logging with pruning of old logs."""
with open(LOG_FILE, 'r') as f: try:
lines = f.readlines() os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True)
cutoff = datetime.now(timezone.utc) - timedelta(days=LOG_PRUNE_DAYS) if not os.access(os.path.dirname(LOG_FILE), os.W_OK):
pruned_lines = [] raise PermissionError(f"No write permission for {os.path.dirname(LOG_FILE)}")
malformed_count = 0 if os.path.exists(LOG_FILE):
for line in lines: with open(LOG_FILE, 'r') as f:
if len(line) < 19 or not line[:19].replace('-', '').replace(':', '').replace(' ', '').isdigit(): lines = f.readlines()
malformed_count += 1 cutoff = datetime.now(timezone.utc) - timedelta(days=LOG_PRUNE_DAYS)
continue pruned_lines = []
try: malformed_count = 0
timestamp = datetime.strptime(line[:19], '%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone.utc) for line in lines:
if timestamp > cutoff: if len(line) < 19 or not line[:19].replace('-', '').replace(':', '').replace(' ', '').isdigit():
pruned_lines.append(line) malformed_count += 1
except ValueError: continue
malformed_count += 1 try:
continue timestamp = datetime.strptime(line[:19], '%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone.utc)
if malformed_count > 0: if timestamp > cutoff:
logging.info(f"Skipped {malformed_count} malformed log lines during pruning") pruned_lines.append(line)
with open(LOG_FILE, 'w') as f: except ValueError:
f.writelines(pruned_lines) malformed_count += 1
continue
logging.basicConfig( if malformed_count > 0:
filename=LOG_FILE, logging.info(f"Skipped {malformed_count} malformed log lines during pruning")
level=logging.INFO, with open(LOG_FILE, 'w') as f:
format="%(asctime)s - %(levelname)s - %(message)s", f.writelines(pruned_lines)
datefmt="%Y-%m-%d %H:%M:%S"
) logging.basicConfig(
console_handler = logging.StreamHandler() filename=LOG_FILE,
console_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) level=logging.INFO,
logging.getLogger().addHandler(console_handler) format="%(asctime)s - %(levelname)s - %(message)s",
logging.getLogger("requests").setLevel(logging.WARNING) datefmt="%Y-%m-%d %H:%M:%S"
logging.info("Logging initialized for foodie_automator_rss.py") )
console_handler = logging.StreamHandler()
console_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
logging.getLogger().addHandler(console_handler)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("openai").setLevel(logging.WARNING)
logging.info("Logging initialized for foodie_automator_rss.py")
except Exception as e:
print(f"Failed to setup logging: {e}")
sys.exit(1)
def acquire_lock(): def acquire_lock():
os.makedirs(os.path.dirname(LOCK_FILE), exist_ok=True) os.makedirs(os.path.dirname(LOCK_FILE), exist_ok=True)
@ -111,6 +109,17 @@ def acquire_lock():
logging.info("Another instance of foodie_automator_rss.py is running") logging.info("Another instance of foodie_automator_rss.py is running")
sys.exit(0) sys.exit(0)
def signal_handler(sig, frame):
logging.info("Received termination signal, checking if safe to exit...")
if is_posting:
logging.info("Currently posting, will exit after completion.")
else:
logging.info("Safe to exit immediately.")
sys.exit(0)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
def create_http_session() -> requests.Session: def create_http_session() -> requests.Session:
session = requests.Session() session = requests.Session()
retry_strategy = Retry( retry_strategy = Retry(
@ -209,7 +218,7 @@ def fetch_duckduckgo_news_context(title, hours=24):
if '+00:00' in date_str: if '+00:00' in date_str:
dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S+00:00").replace(tzinfo=timezone.utc) dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S+00:00").replace(tzinfo=timezone.utc)
else: else:
dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S%Z").replace(tzinfo=timezone.utc) dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
if dt > (datetime.now(timezone.utc) - timedelta(hours=24)): if dt > (datetime.now(timezone.utc) - timedelta(hours=24)):
titles.append(r["title"].lower()) titles.append(r["title"].lower())
except ValueError as e: except ValueError as e:
@ -324,6 +333,10 @@ def curate_from_rss():
interest_score=interest_score, interest_score=interest_score,
should_post_tweet=True should_post_tweet=True
) )
if not post_id:
logging.warning(f"Failed to post to WordPress for '{title}', skipping")
attempts += 1
continue
except Exception as e: except Exception as e:
logging.error(f"Failed to post to WordPress for '{title}': {e}", exc_info=True) logging.error(f"Failed to post to WordPress for '{title}': {e}", exc_info=True)
attempts += 1 attempts += 1
@ -383,6 +396,7 @@ def run_rss_automator():
lock_fd = None lock_fd = None
try: try:
lock_fd = acquire_lock() lock_fd = acquire_lock()
setup_logging()
logging.info("***** RSS Automator Launched *****") logging.info("***** RSS Automator Launched *****")
post_data, category, should_continue = curate_from_rss() post_data, category, should_continue = curate_from_rss()
if not post_data: if not post_data:
@ -392,6 +406,7 @@ def run_rss_automator():
return post_data, category, should_continue return post_data, category, should_continue
except Exception as e: except Exception as e:
logging.error(f"Fatal error in run_rss_automator: {e}", exc_info=True) logging.error(f"Fatal error in run_rss_automator: {e}", exc_info=True)
print(f"Fatal error: {e}")
return None, None, False return None, None, False
finally: finally:
if lock_fd: if lock_fd:
@ -400,7 +415,4 @@ def run_rss_automator():
os.remove(LOCK_FILE) if os.path.exists(LOCK_FILE) else None os.remove(LOCK_FILE) if os.path.exists(LOCK_FILE) else None
if __name__ == "__main__": if __name__ == "__main__":
setup_logging() run_rss_automator()
post_data, category, should_continue = run_rss_automator()
# Remove sleep timer, let manage_scripts.sh control execution
logging.info(f"Run completed, should_continue: {should_continue}")

26
foodie_utils.py
Unescape View File

@ -448,25 +448,24 @@ def upload_image_to_wp(image_url, post_title, wp_base_url, wp_username, wp_passw
} }
logging.info(f"Fetching image from {image_url} for '{post_title}'") logging.info(f"Fetching image from {image_url} for '{post_title}'")
for attempt in range(3): for attempt in range(MAX_RETRIES):
try: try:
image_response = requests.get(image_url, headers=image_headers, timeout=10) image_response = requests.get(image_url, headers=image_headers, timeout=IMAGE_UPLOAD_TIMEOUT)
if image_response.status_code == 429: if image_response.status_code == 429:
wait_time = 10 * (2 ** attempt) wait_time = RETRY_BACKOFF * (2 ** attempt)
logging.warning(f"Rate limit hit for {image_url}. Retrying after {wait_time}s (attempt {attempt+1}/3).") logging.warning(f"Rate limit hit for {image_url}. Retrying after {wait_time}s (attempt {attempt+1}/{MAX_RETRIES}).")
time.sleep(wait_time) time.sleep(wait_time)
continue continue
image_response.raise_for_status() image_response.raise_for_status()
break break
except requests.exceptions.HTTPError as e: except requests.exceptions.RequestException as e:
if e.response.status_code == 429: logging.warning(f"Image fetch failed for {image_url} (attempt {attempt+1}/{MAX_RETRIES}): {e}")
wait_time = 10 * (2 ** attempt) if attempt == MAX_RETRIES - 1:
logging.warning(f"Rate limit hit for {image_url}. Retrying after {wait_time}s (attempt {attempt+1}/3).") logging.error(f"Failed to fetch image {image_url} after {MAX_RETRIES} attempts")
time.sleep(wait_time) return None
continue time.sleep(RETRY_BACKOFF * (2 ** attempt))
raise
else: else:
logging.warning(f"Rate limit hit for {image_url} after retries. Failing image upload.") logging.error(f"Failed to fetch image {image_url} after retries")
return None return None
response = requests.post( response = requests.post(
@ -492,7 +491,8 @@ def upload_image_to_wp(image_url, post_title, wp_base_url, wp_username, wp_passw
logging.info(f"Uploaded image '{safe_title}.jpg' to WP (ID: {image_id}) with caption '{caption}'") logging.info(f"Uploaded image '{safe_title}.jpg' to WP (ID: {image_id}) with caption '{caption}'")
return image_id return image_id
except Exception as e: except Exception as e:
logging.error(f"Image upload to WP failed for '{post_title}': {e}") logging.error(f"Image upload to WP failed for '{post_title}': {e}", exc_info=True)
print(f"Image upload to WP failed for '{post_title}': {e}")
return None return None
def determine_paragraph_count(interest_score): def determine_paragraph_count(interest_score):

Write Preview
Loading…
Cancel
Save