|
|
|
|
@ -29,63 +29,62 @@ from foodie_config import ( |
|
|
|
|
load_dotenv() |
|
|
|
|
client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) |
|
|
|
|
|
|
|
|
|
def load_json_file(filename, expiration_days=None): |
|
|
|
|
data = [] |
|
|
|
|
if os.path.exists(filename): |
|
|
|
|
def load_json_file(file_path, expiration_hours): |
|
|
|
|
entries = [] |
|
|
|
|
cutoff = datetime.now(timezone.utc) - timedelta(hours=expiration_hours) |
|
|
|
|
|
|
|
|
|
if not os.path.exists(file_path): |
|
|
|
|
logging.info(f"File {file_path} does not exist, returning empty list") |
|
|
|
|
return entries |
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
with open(filename, 'r') as f: |
|
|
|
|
with open(file_path, 'r') as f: |
|
|
|
|
lines = f.readlines() |
|
|
|
|
|
|
|
|
|
for i, line in enumerate(lines, 1): |
|
|
|
|
if line.strip(): |
|
|
|
|
try: |
|
|
|
|
entry = json.loads(line.strip()) |
|
|
|
|
if not isinstance(entry, dict) or "title" not in entry or "timestamp" not in entry: |
|
|
|
|
logging.warning(f"Skipping malformed entry in {filename} at line {i}: {entry}") |
|
|
|
|
logging.warning(f"Skipping malformed entry in {file_path} at line {i}: {line.strip()}") |
|
|
|
|
continue |
|
|
|
|
data.append(entry) |
|
|
|
|
|
|
|
|
|
timestamp = datetime.fromisoformat(entry["timestamp"]) |
|
|
|
|
if timestamp > cutoff: |
|
|
|
|
entries.append(entry) |
|
|
|
|
else: |
|
|
|
|
logging.debug(f"Entry expired in {file_path}: {entry['title']}") |
|
|
|
|
except json.JSONDecodeError as e: |
|
|
|
|
logging.warning(f"Skipping invalid JSON line in {filename} at line {i}: {e}") |
|
|
|
|
if expiration_days: |
|
|
|
|
cutoff = (datetime.now(timezone.utc) - timedelta(days=expiration_days)).isoformat() |
|
|
|
|
data = [entry for entry in data if entry["timestamp"] > cutoff] |
|
|
|
|
logging.info(f"Loaded {len(data)} entries from {filename}, {len(data)} valid after expiration check") |
|
|
|
|
logging.warning(f"Skipping invalid JSON line in {file_path} at line {i}: {e}") |
|
|
|
|
continue |
|
|
|
|
except Exception as e: |
|
|
|
|
logging.error(f"Failed to load {filename}: {e}") |
|
|
|
|
data = [] # Reset to empty on failure |
|
|
|
|
return data |
|
|
|
|
logging.warning(f"Skipping malformed entry in {file_path} at line {i}: {line.strip()}") |
|
|
|
|
continue |
|
|
|
|
|
|
|
|
|
def save_json_file(filename, key, value): |
|
|
|
|
entry = {"title": key, "timestamp": value} |
|
|
|
|
PRUNE_INTERVAL_DAYS = 180 |
|
|
|
|
logging.info(f"Loaded {len(entries)} entries from {file_path}, {len(entries)} valid after expiration check") |
|
|
|
|
return entries |
|
|
|
|
except Exception as e: |
|
|
|
|
logging.error(f"Failed to load {file_path}: {e}") |
|
|
|
|
return entries |
|
|
|
|
|
|
|
|
|
def save_json_file(file_path, title, timestamp): |
|
|
|
|
try: |
|
|
|
|
data = load_json_file(filename, expiration_days=PRUNE_INTERVAL_DAYS) |
|
|
|
|
# Remove duplicates by title |
|
|
|
|
data = [item for item in data if item["title"] != key] |
|
|
|
|
data.append(entry) |
|
|
|
|
# Special handling for used_images.json to save as a flat list with one URL per line |
|
|
|
|
if filename.endswith('used_images.json'): |
|
|
|
|
used_images.add(key) |
|
|
|
|
with open(filename, 'w') as f: |
|
|
|
|
f.write('[\n') |
|
|
|
|
urls = list(used_images) |
|
|
|
|
for i, url in enumerate(urls): |
|
|
|
|
f.write(f'"{url}"') |
|
|
|
|
if i < len(urls) - 1: |
|
|
|
|
f.write(',\n') |
|
|
|
|
else: |
|
|
|
|
f.write('\n') |
|
|
|
|
f.write(']') |
|
|
|
|
else: |
|
|
|
|
with open(filename, 'w') as f: |
|
|
|
|
for item in data: |
|
|
|
|
json.dump(item, f) |
|
|
|
|
f.write('\n') |
|
|
|
|
logging.info(f"Saved '{key}' to {filename}") |
|
|
|
|
print(f"DEBUG: Saved '{key}' to {filename}") |
|
|
|
|
loaded_data = load_json_file(filename, expiration_days=PRUNE_INTERVAL_DAYS) |
|
|
|
|
logging.info(f"Pruned {filename} to {len(loaded_data)} entries (older than {PRUNE_INTERVAL_DAYS} days removed)") |
|
|
|
|
entries = load_json_file(file_path, 24 if "posted_" in file_path else 7 * 24) # 24 hours for titles, 7 days for images |
|
|
|
|
entry = {"title": title, "timestamp": timestamp} |
|
|
|
|
entries.append(entry) |
|
|
|
|
|
|
|
|
|
# Prune entries older than expiration period |
|
|
|
|
expiration_hours = 24 if "posted_" in file_path else 7 * 24 |
|
|
|
|
cutoff = datetime.now(timezone.utc) - timedelta(hours=expiration_hours) |
|
|
|
|
pruned_entries = [e for e in entries if datetime.fromisoformat(e["timestamp"]) > cutoff] |
|
|
|
|
|
|
|
|
|
with open(file_path, 'w') as f: |
|
|
|
|
for entry in pruned_entries: |
|
|
|
|
f.write(json.dumps(entry) + '\n') |
|
|
|
|
|
|
|
|
|
logging.info(f"Saved '{title}' to {file_path}") |
|
|
|
|
logging.info(f"Pruned {file_path} to {len(pruned_entries)} entries (older than {expiration_hours//24} days removed)") |
|
|
|
|
except Exception as e: |
|
|
|
|
logging.error(f"Failed to save or prune {filename}: {e}") |
|
|
|
|
logging.error(f"Failed to save to {file_path}: {e}") |
|
|
|
|
|
|
|
|
|
def load_post_counts(): |
|
|
|
|
counts = [] |
|
|
|
|
@ -887,6 +886,7 @@ def post_to_wp(post_data, category, link, author, image_url, original_source, im |
|
|
|
|
logging.error(f"Post content is None for title '{post_data['title']}' - using fallback") |
|
|
|
|
content = "Content unavailable. Check the original source for details." |
|
|
|
|
formatted_content = "\n".join(f"<p>{para}</p>" for para in content.split('\n') if para.strip()) |
|
|
|
|
|
|
|
|
|
author_id_map = { |
|
|
|
|
"owenjohnson": 10, |
|
|
|
|
"javiermorales": 2, |
|
|
|
|
@ -897,6 +897,20 @@ def post_to_wp(post_data, category, link, author, image_url, original_source, im |
|
|
|
|
} |
|
|
|
|
author_id = author_id_map.get(author["username"], 5) |
|
|
|
|
|
|
|
|
|
# Handle image upload |
|
|
|
|
image_id = None |
|
|
|
|
if image_url: |
|
|
|
|
logging.info(f"Attempting image upload for '{post_data['title']}', URL: {image_url}, source: {image_source}") |
|
|
|
|
image_id = upload_image_to_wp(image_url, post_data["title"], wp_base_url, wp_username, wp_password, image_source, uploader, pixabay_url) |
|
|
|
|
if not image_id: |
|
|
|
|
logging.info(f"Flickr upload failed for '{post_data['title']}', falling back to Pixabay") |
|
|
|
|
pixabay_query = post_data["title"][:50] |
|
|
|
|
image_url, image_source, uploader, pixabay_url = get_image(pixabay_query) |
|
|
|
|
if image_url: |
|
|
|
|
image_id = upload_image_to_wp(image_url, post_data["title"], wp_base_url, wp_username, wp_password, image_source, uploader, pixabay_url) |
|
|
|
|
if not image_id: |
|
|
|
|
logging.warning(f"All image uploads failed for '{post_data['title']}' - posting without image") |
|
|
|
|
|
|
|
|
|
payload = { |
|
|
|
|
"title": post_data["title"], |
|
|
|
|
"content": formatted_content, |
|
|
|
|
@ -911,19 +925,9 @@ def post_to_wp(post_data, category, link, author, image_url, original_source, im |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if image_url and not post_id: |
|
|
|
|
logging.info(f"Attempting image upload for '{post_data['title']}', URL: {image_url}, source: {image_source}") |
|
|
|
|
image_id = upload_image_to_wp(image_url, post_data["title"], wp_base_url, wp_username, wp_password, image_source, uploader, pixabay_url) |
|
|
|
|
if not image_id: |
|
|
|
|
logging.info(f"Flickr upload failed for '{post_data['title']}', falling back to Pixabay") |
|
|
|
|
pixabay_query = post_data["title"][:50] |
|
|
|
|
image_url, image_source, uploader, pixabay_url = get_image(pixabay_query) |
|
|
|
|
if image_url: |
|
|
|
|
image_id = upload_image_to_wp(image_url, post_data["title"], wp_base_url, wp_username, wp_password, image_source, uploader, pixabay_url) |
|
|
|
|
if image_id: |
|
|
|
|
payload["featured_media"] = image_id |
|
|
|
|
else: |
|
|
|
|
logging.warning(f"All image uploads failed for '{post_data['title']}' - posting without image") |
|
|
|
|
logging.info(f"Set featured image for post '{post_data['title']}': Media ID={image_id}") |
|
|
|
|
|
|
|
|
|
endpoint = f"{wp_base_url}/posts/{post_id}" if post_id else f"{wp_base_url}/posts" |
|
|
|
|
method = requests.post |
|
|
|
|
@ -951,7 +955,7 @@ def post_to_wp(post_data, category, link, author, image_url, original_source, im |
|
|
|
|
try: |
|
|
|
|
post = {"title": post_data["title"], "url": post_url} |
|
|
|
|
tweet = generate_article_tweet(author, post, author["persona"]) |
|
|
|
|
if post_tweet(author, tweet): # Use the actual post_tweet function |
|
|
|
|
if post_tweet(author, tweet): |
|
|
|
|
logging.info(f"Successfully posted article tweet for {author['username']} on X") |
|
|
|
|
else: |
|
|
|
|
logging.warning(f"Failed to post article tweet for {author['username']} on X") |
|
|
|
|
|
