add rate limiter

foodie_utils.py

@@ -29,7 +29,7 @@ from typing import List, Dict, Any, Optional, Union, Tuple
 from pathlib import Path
 from functools import lru_cache
 import hashlib
-from rate_limiter import RateLimiter
+from .rate_limiter import RateLimiter
 
 # Configure logging
 logging.basicConfig(

rate_limiter.py

@@ -0,0 +1,57 @@
+# rate_limiter.py
+import time
+from datetime import datetime, timedelta
+from typing import Optional
+import logging
+
+logger = logging.getLogger(__name__)
+
+class RateLimiter:
+    """A rate limiter that enforces a maximum number of requests per time period."""
+    
+    def __init__(self, max_requests: int, time_window: int):
+        """
+        Initialize the rate limiter.
+        
+        Args:
+            max_requests: Maximum number of requests allowed in the time window
+            time_window: Time window in seconds
+        """
+        self.max_requests = max_requests
+        self.time_window = time_window
+        self.requests = []
+        self.last_cleanup = datetime.now()
+        
+    def _cleanup_old_requests(self) -> None:
+        """Remove requests older than the time window."""
+        now = datetime.now()
+        if (now - self.last_cleanup).total_seconds() > self.time_window:
+            cutoff = now - timedelta(seconds=self.time_window)
+            self.requests = [req for req in self.requests if req > cutoff]
+            self.last_cleanup = now
+            
+    def wait_if_needed(self) -> Optional[float]:
+        """
+        Wait if necessary to respect the rate limit.
+        
+        Returns:
+            Optional[float]: The number of seconds waited, or None if no wait was needed
+        """
+        self._cleanup_old_requests()
+        
+        if len(self.requests) >= self.max_requests:
+            oldest_request = self.requests[0]
+            wait_time = (datetime.now() - oldest_request).total_seconds()
+            if wait_time < self.time_window:
+                sleep_time = self.time_window - wait_time
+                logger.info(f"Rate limit reached. Waiting {sleep_time:.2f} seconds...")
+                time.sleep(sleep_time)
+                return sleep_time
+                
+        self.requests.append(datetime.now())
+        return None
+        
+    def reset(self) -> None:
+        """Reset the rate limiter's request history."""
+        self.requests = []
+        self.last_cleanup = datetime.now() 

requirements.txt

@@ -1,13 +1,30 @@
-requests==2.32.3
+# Core dependencies
-selenium==4.29.0
+requests>=2.32.3,<3.0.0
-duckduckgo_search==7.5.4
+python-dotenv>=1.0.1,<2.0.0
-openai==1.75.0
+urllib3>=2.0.0,<3.0.0
-praw==7.8.1
+
-beautifulsoup4==4.13.3
+# Web scraping and automation
-Pillow==11.1.0
+selenium>=4.29.0,<5.0.0
-pytesseract==0.3.13
+beautifulsoup4>=4.13.3,<5.0.0
-feedparser==6.0.11
+feedparser>=6.0.11,<7.0.0
-webdriver-manager==4.0.2
+webdriver-manager>=4.0.2,<5.0.0
-tweepy==4.14.0
+duckduckgo_search>=7.5.4,<8.0.0
-python-dotenv==1.0.1
+
-flickr-api==0.7.1
+# API clients
+openai>=1.75.0,<2.0.0
+praw>=7.8.1,<8.0.0
+tweepy>=4.14.0,<5.0.0
+flickr-api>=0.7.1,<1.0.0
+
+# Image processing
+Pillow>=11.1.0,<12.0.0
+pytesseract>=0.3.13,<1.0.0
+
+# Development tools
+black>=24.1.1,<25.0.0
+flake8>=7.0.0,<8.0.0
+mypy>=1.8.0,<2.0.0
+pytest>=8.0.0,<9.0.0
+
+# WordPress integration
+python-wordpress-xmlrpc>=2.3,<3.0