diff --git a/foodie_utils.py b/foodie_utils.py index 4cc50f1..edc4f54 100644 --- a/foodie_utils.py +++ b/foodie_utils.py @@ -29,7 +29,7 @@ from typing import List, Dict, Any, Optional, Union, Tuple from pathlib import Path from functools import lru_cache import hashlib -from rate_limiter import RateLimiter +from .rate_limiter import RateLimiter # Configure logging logging.basicConfig( diff --git a/rate_limiter.py b/rate_limiter.py new file mode 100644 index 0000000..6884779 --- /dev/null +++ b/rate_limiter.py @@ -0,0 +1,57 @@ +# rate_limiter.py +import time +from datetime import datetime, timedelta +from typing import Optional +import logging + +logger = logging.getLogger(__name__) + +class RateLimiter: + """A rate limiter that enforces a maximum number of requests per time period.""" + + def __init__(self, max_requests: int, time_window: int): + """ + Initialize the rate limiter. + + Args: + max_requests: Maximum number of requests allowed in the time window + time_window: Time window in seconds + """ + self.max_requests = max_requests + self.time_window = time_window + self.requests = [] + self.last_cleanup = datetime.now() + + def _cleanup_old_requests(self) -> None: + """Remove requests older than the time window.""" + now = datetime.now() + if (now - self.last_cleanup).total_seconds() > self.time_window: + cutoff = now - timedelta(seconds=self.time_window) + self.requests = [req for req in self.requests if req > cutoff] + self.last_cleanup = now + + def wait_if_needed(self) -> Optional[float]: + """ + Wait if necessary to respect the rate limit. + + Returns: + Optional[float]: The number of seconds waited, or None if no wait was needed + """ + self._cleanup_old_requests() + + if len(self.requests) >= self.max_requests: + oldest_request = self.requests[0] + wait_time = (datetime.now() - oldest_request).total_seconds() + if wait_time < self.time_window: + sleep_time = self.time_window - wait_time + logger.info(f"Rate limit reached. Waiting {sleep_time:.2f} seconds...") + time.sleep(sleep_time) + return sleep_time + + self.requests.append(datetime.now()) + return None + + def reset(self) -> None: + """Reset the rate limiter's request history.""" + self.requests = [] + self.last_cleanup = datetime.now() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b58c5c3..2f01552 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,30 @@ -requests==2.32.3 -selenium==4.29.0 -duckduckgo_search==7.5.4 -openai==1.75.0 -praw==7.8.1 -beautifulsoup4==4.13.3 -Pillow==11.1.0 -pytesseract==0.3.13 -feedparser==6.0.11 -webdriver-manager==4.0.2 -tweepy==4.14.0 -python-dotenv==1.0.1 -flickr-api==0.7.1 \ No newline at end of file +# Core dependencies +requests>=2.32.3,<3.0.0 +python-dotenv>=1.0.1,<2.0.0 +urllib3>=2.0.0,<3.0.0 + +# Web scraping and automation +selenium>=4.29.0,<5.0.0 +beautifulsoup4>=4.13.3,<5.0.0 +feedparser>=6.0.11,<7.0.0 +webdriver-manager>=4.0.2,<5.0.0 +duckduckgo_search>=7.5.4,<8.0.0 + +# API clients +openai>=1.75.0,<2.0.0 +praw>=7.8.1,<8.0.0 +tweepy>=4.14.0,<5.0.0 +flickr-api>=0.7.1,<1.0.0 + +# Image processing +Pillow>=11.1.0,<12.0.0 +pytesseract>=0.3.13,<1.0.0 + +# Development tools +black>=24.1.1,<25.0.0 +flake8>=7.0.0,<8.0.0 +mypy>=1.8.0,<2.0.0 +pytest>=8.0.0,<9.0.0 + +# WordPress integration +python-wordpress-xmlrpc>=2.3,<3.0 \ No newline at end of file