diff --git a/src/brightdata/api/async_unblocker.py b/src/brightdata/api/async_unblocker.py index f1fc80a..3b6d295 100644 --- a/src/brightdata/api/async_unblocker.py +++ b/src/brightdata/api/async_unblocker.py @@ -70,7 +70,7 @@ async def trigger( zone: str, url: str, customer: Optional[str] = None, - **kwargs # Additional params like country, format, etc. + **kwargs, # Additional params like country, format, etc. ) -> Optional[str]: """ Trigger async unblocker request. @@ -105,21 +105,14 @@ async def trigger( payload.update(kwargs) async with self.engine.post_to_url( - f"{self.engine.BASE_URL}{self.TRIGGER_ENDPOINT}", - params=params, - json_data=payload + f"{self.engine.BASE_URL}{self.TRIGGER_ENDPOINT}", params=params, json_data=payload ) as response: # Extract response_id from x-response-id header # Note: This is different from datasets API which returns snapshot_id in body response_id = response.headers.get("x-response-id") return response_id - async def get_status( - self, - zone: str, - response_id: str, - customer: Optional[str] = None - ) -> str: + async def get_status(self, zone: str, response_id: str, customer: Optional[str] = None) -> str: """ Check if response is ready. @@ -142,18 +135,14 @@ async def get_status( >>> if status == "ready": ... # Fetch results """ - params = { - "zone": zone, - "response_id": response_id - } + params = {"zone": zone, "response_id": response_id} # Add customer to query params if provided if customer: params["customer"] = customer async with self.engine.get_from_url( - f"{self.engine.BASE_URL}{self.FETCH_ENDPOINT}", - params=params + f"{self.engine.BASE_URL}{self.FETCH_ENDPOINT}", params=params ) as response: if response.status == 200: return "ready" @@ -168,7 +157,7 @@ async def fetch_result( zone: str, response_id: str, response_format: str = "json", - customer: Optional[str] = None + customer: Optional[str] = None, ) -> Any: """ Fetch results when ready. @@ -203,18 +192,14 @@ async def fetch_result( ... customer="hl_67e5ed38" ... ) """ - params = { - "zone": zone, - "response_id": response_id - } + params = {"zone": zone, "response_id": response_id} # Add customer to query params if provided if customer: params["customer"] = customer async with self.engine.get_from_url( - f"{self.engine.BASE_URL}{self.FETCH_ENDPOINT}", - params=params + f"{self.engine.BASE_URL}{self.FETCH_ENDPOINT}", params=params ) as response: if response.status == 200: # Success - parse based on format diff --git a/src/brightdata/api/scrape_service.py b/src/brightdata/api/scrape_service.py index 86d721b..9c27087 100644 --- a/src/brightdata/api/scrape_service.py +++ b/src/brightdata/api/scrape_service.py @@ -182,5 +182,3 @@ def instagram(self): bearer_token=self._client.token, engine=self._client.engine ) return self._instagram - - diff --git a/src/brightdata/api/search_service.py b/src/brightdata/api/search_service.py index 6149919..fa43d3d 100644 --- a/src/brightdata/api/search_service.py +++ b/src/brightdata/api/search_service.py @@ -103,7 +103,6 @@ async def google( **kwargs, ) - async def bing( self, query: Union[str, List[str]], @@ -132,7 +131,6 @@ async def bing( **kwargs, ) - async def yandex( self, query: Union[str, List[str]], @@ -161,7 +159,6 @@ async def yandex( **kwargs, ) - @property def amazon(self): """ diff --git a/src/brightdata/api/serp/base.py b/src/brightdata/api/serp/base.py index 962c72a..37b35c1 100644 --- a/src/brightdata/api/serp/base.py +++ b/src/brightdata/api/serp/base.py @@ -153,7 +153,6 @@ async def search( **kwargs, ) - async def _search_single_async( self, query: str, diff --git a/src/brightdata/api/web_unlocker.py b/src/brightdata/api/web_unlocker.py index 4830c22..c9c9b7a 100644 --- a/src/brightdata/api/web_unlocker.py +++ b/src/brightdata/api/web_unlocker.py @@ -308,7 +308,7 @@ async def _scrape_single_async_unblocker( url=url, format=response_format, method=method, - country=country.upper() if country else None + country=country.upper() if country else None, ) except Exception as e: return ScrapeResult( @@ -370,9 +370,7 @@ async def _scrape_single_async_unblocker( try: data = await self.async_unblocker.fetch_result( - zone, - response_id, - response_format=response_format + zone, response_id, response_format=response_format ) root_domain = extract_root_domain(url) diff --git a/src/brightdata/cli/commands/scrape.py b/src/brightdata/cli/commands/scrape.py index 3b01836..be1d4e4 100644 --- a/src/brightdata/cli/commands/scrape.py +++ b/src/brightdata/cli/commands/scrape.py @@ -50,9 +50,7 @@ def scrape_url(ctx: click.Context, url: str, country: str, response_format: str) """Scrape any URL using Web Unlocker.""" try: client = create_client(ctx.obj["api_key"]) - result = client.scrape_url( - url=url, country=country, response_format=response_format - ) + result = client.scrape_url(url=url, country=country, response_format=response_format) output_result(result, ctx.obj["output_format"], ctx.obj["output_file"]) except Exception as e: handle_error(e) diff --git a/src/brightdata/client.py b/src/brightdata/client.py index 6beae32..253ae5d 100644 --- a/src/brightdata/client.py +++ b/src/brightdata/client.py @@ -507,7 +507,6 @@ async def scrape_url( poll_timeout=poll_timeout, ) - async def __aenter__(self): """Async context manager entry.""" await self.engine.__aenter__() @@ -517,9 +516,7 @@ async def __aenter__(self): is_valid = await self.test_connection() if not is_valid: await self.engine.__aexit__(None, None, None) - raise AuthenticationError( - "Token validation failed. Please check your API token." - ) + raise AuthenticationError("Token validation failed. Please check your API token.") await self._ensure_zones() return self @@ -533,5 +530,3 @@ def __repr__(self) -> str: token_preview = f"{self.token[:10]}...{self.token[-5:]}" if self.token else "None" status = "Connected" if self._is_connected else "Not tested" return f"" - - diff --git a/src/brightdata/core/engine.py b/src/brightdata/core/engine.py index 6f72949..40be1d8 100644 --- a/src/brightdata/core/engine.py +++ b/src/brightdata/core/engine.py @@ -24,7 +24,9 @@ warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.* ScrapeJob: """Trigger Amazon products scrape (sync version).""" return asyncio.run(self.products_trigger(url)) @@ -159,7 +159,6 @@ async def products_status(self, snapshot_id: str) -> str: """ return await self._check_status_async(snapshot_id) - def products_status_sync(self, snapshot_id: str) -> str: """Check Amazon products scrape status (sync version).""" return asyncio.run(self.products_status(snapshot_id)) @@ -179,7 +178,6 @@ async def products_fetch(self, snapshot_id: str) -> Any: """ return await self._fetch_results_async(snapshot_id) - def products_fetch_sync(self, snapshot_id: str) -> Any: """Fetch Amazon products scrape results (sync version).""" return asyncio.run(self.products_fetch(snapshot_id)) @@ -276,7 +274,6 @@ async def reviews( return results return result - def reviews_sync( self, url: Union[str, List[str]], @@ -290,9 +287,11 @@ def reviews_sync( See reviews() for full documentation. """ + async def _run(): async with self.engine: return await self.reviews(url, pastDays, keyWord, numOfReviews, timeout) + return asyncio.run(_run()) # ============================================================================ @@ -332,7 +331,6 @@ async def reviews_trigger( sdk_function=sdk_function or "reviews_trigger", ) - def reviews_trigger_sync( self, url: Union[str, List[str]], @@ -347,7 +345,6 @@ async def reviews_status(self, snapshot_id: str) -> str: """Check Amazon reviews scrape status.""" return await self._check_status_async(snapshot_id) - def reviews_status_sync(self, snapshot_id: str) -> str: """Check Amazon reviews scrape status (sync version).""" return asyncio.run(self.reviews_status(snapshot_id)) @@ -356,7 +353,6 @@ async def reviews_fetch(self, snapshot_id: str) -> Any: """Fetch Amazon reviews scrape results.""" return await self._fetch_results_async(snapshot_id) - def reviews_fetch_sync(self, snapshot_id: str) -> Any: """Fetch Amazon reviews scrape results (sync version).""" return asyncio.run(self.reviews_fetch(snapshot_id)) @@ -397,7 +393,6 @@ async def sellers( return await self._scrape_urls(url=url, dataset_id=self.DATASET_ID_SELLERS, timeout=timeout) - def sellers_sync( self, url: Union[str, List[str]], @@ -408,9 +403,11 @@ def sellers_sync( See sellers() for full documentation. """ + async def _run(): async with self.engine: return await self.sellers(url, timeout) + return asyncio.run(_run()) # ============================================================================ @@ -444,7 +441,6 @@ async def sellers_trigger( sdk_function=sdk_function or "sellers_trigger", ) - def sellers_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: """Trigger Amazon sellers scrape (sync version).""" return asyncio.run(self.sellers_trigger(url)) @@ -453,7 +449,6 @@ async def sellers_status(self, snapshot_id: str) -> str: """Check Amazon sellers scrape status.""" return await self._check_status_async(snapshot_id) - def sellers_status_sync(self, snapshot_id: str) -> str: """Check Amazon sellers scrape status (sync version).""" return asyncio.run(self.sellers_status(snapshot_id)) @@ -462,7 +457,6 @@ async def sellers_fetch(self, snapshot_id: str) -> Any: """Fetch Amazon sellers scrape results.""" return await self._fetch_results_async(snapshot_id) - def sellers_fetch_sync(self, snapshot_id: str) -> Any: """Fetch Amazon sellers scrape results (sync version).""" return asyncio.run(self.sellers_fetch(snapshot_id)) diff --git a/src/brightdata/scrapers/amazon/search.py b/src/brightdata/scrapers/amazon/search.py index dde58b4..955ec6b 100644 --- a/src/brightdata/scrapers/amazon/search.py +++ b/src/brightdata/scrapers/amazon/search.py @@ -164,7 +164,6 @@ async def products( timeout=timeout, ) - def products_sync( self, keyword: Optional[Union[str, List[str]]] = None, @@ -182,6 +181,7 @@ def products_sync( See products() for full documentation. """ + async def _run(): async with self.engine: return await self.products( @@ -195,6 +195,7 @@ async def _run(): country=country, timeout=timeout, ) + return asyncio.run(_run()) # ============================================================================ diff --git a/src/brightdata/scrapers/chatgpt/scraper.py b/src/brightdata/scrapers/chatgpt/scraper.py index eb159e6..d98cce4 100644 --- a/src/brightdata/scrapers/chatgpt/scraper.py +++ b/src/brightdata/scrapers/chatgpt/scraper.py @@ -120,7 +120,6 @@ async def prompt( return result - def prompt_sync( self, prompt: str, @@ -138,6 +137,7 @@ def prompt_sync( Example: >>> result = scraper.prompt_sync("Explain Python asyncio") """ + async def _run(): async with self.engine: return await self.prompt( @@ -191,7 +191,6 @@ async def prompt_trigger( cost_per_record=self.COST_PER_RECORD, ) - def prompt_trigger_sync( self, prompt: str, @@ -200,33 +199,37 @@ def prompt_trigger_sync( additional_prompt: Optional[str] = None, ) -> "ScrapeJob": """Trigger ChatGPT prompt (sync wrapper).""" + async def _run(): async with self.engine: return await self.prompt_trigger(prompt, country, web_search, additional_prompt) + return asyncio.run(_run()) async def prompt_status(self, snapshot_id: str) -> str: """Check ChatGPT prompt status (async).""" return await self._check_status_async(snapshot_id) - def prompt_status_sync(self, snapshot_id: str) -> str: """Check ChatGPT prompt status (sync wrapper).""" + async def _run(): async with self.engine: return await self.prompt_status(snapshot_id) + return asyncio.run(_run()) async def prompt_fetch(self, snapshot_id: str) -> Any: """Fetch ChatGPT prompt results (async).""" return await self._fetch_results_async(snapshot_id) - def prompt_fetch_sync(self, snapshot_id: str) -> Any: """Fetch ChatGPT prompt results (sync wrapper).""" + async def _run(): async with self.engine: return await self.prompt_fetch(snapshot_id) + return asyncio.run(_run()) # ============================================================================ @@ -300,7 +303,6 @@ async def prompts( return result - def prompts_sync( self, prompts: List[str], @@ -315,6 +317,7 @@ def prompts_sync( See prompts() for full documentation. """ + async def _run(): async with self.engine: return await self.prompts( @@ -368,7 +371,6 @@ async def prompts_trigger( cost_per_record=self.COST_PER_RECORD, ) - def prompts_trigger_sync( self, prompts: List[str], @@ -377,33 +379,39 @@ def prompts_trigger_sync( additional_prompts: Optional[List[str]] = None, ) -> "ScrapeJob": """Trigger ChatGPT batch prompts (sync wrapper).""" + async def _run(): async with self.engine: - return await self.prompts_trigger(prompts, countries, web_searches, additional_prompts) + return await self.prompts_trigger( + prompts, countries, web_searches, additional_prompts + ) + return asyncio.run(_run()) async def prompts_status(self, snapshot_id: str) -> str: """Check ChatGPT batch prompts status (async).""" return await self._check_status_async(snapshot_id) - def prompts_status_sync(self, snapshot_id: str) -> str: """Check ChatGPT batch prompts status (sync wrapper).""" + async def _run(): async with self.engine: return await self.prompts_status(snapshot_id) + return asyncio.run(_run()) async def prompts_fetch(self, snapshot_id: str) -> Any: """Fetch ChatGPT batch prompts results (async).""" return await self._fetch_results_async(snapshot_id) - def prompts_fetch_sync(self, snapshot_id: str) -> Any: """Fetch ChatGPT batch prompts results (sync wrapper).""" + async def _run(): async with self.engine: return await self.prompts_fetch(snapshot_id) + return asyncio.run(_run()) # ============================================================================ @@ -423,7 +431,6 @@ async def scrape( "Use prompt() or prompts() methods instead." ) - def scrape_sync(self, urls: Union[str, List[str]], **kwargs): """ChatGPT doesn't support URL-based scraping.""" raise NotImplementedError( diff --git a/src/brightdata/scrapers/chatgpt/search.py b/src/brightdata/scrapers/chatgpt/search.py index a3b5914..5a6e5ae 100644 --- a/src/brightdata/scrapers/chatgpt/search.py +++ b/src/brightdata/scrapers/chatgpt/search.py @@ -150,7 +150,6 @@ async def chatGPT( return result - def chatGPT_sync( self, prompt: Union[str, List[str]], @@ -170,6 +169,7 @@ def chatGPT_sync( ... webSearch=True ... ) """ + async def _run(): async with self.engine: return await self.chatGPT( @@ -179,6 +179,7 @@ async def _run(): webSearch=webSearch, timeout=timeout, ) + return asyncio.run(_run()) # ============================================================================ diff --git a/src/brightdata/scrapers/facebook/scraper.py b/src/brightdata/scrapers/facebook/scraper.py index a025bd7..6473547 100644 --- a/src/brightdata/scrapers/facebook/scraper.py +++ b/src/brightdata/scrapers/facebook/scraper.py @@ -278,9 +278,7 @@ async def _run(): # --- Trigger Interface (Manual Control) --- - async def posts_by_group_trigger( - self, url: Union[str, List[str]], **kwargs - ) -> "ScrapeJob": + async def posts_by_group_trigger(self, url: Union[str, List[str]], **kwargs) -> "ScrapeJob": """Trigger Facebook posts by group scrape (async - manual control).""" from ..job import ScrapeJob diff --git a/src/brightdata/scrapers/job.py b/src/brightdata/scrapers/job.py index bf02391..9534639 100644 --- a/src/brightdata/scrapers/job.py +++ b/src/brightdata/scrapers/job.py @@ -97,7 +97,6 @@ async def status(self, refresh: bool = True) -> str: self._cached_status = await self._api_client.get_status(self.snapshot_id) return self._cached_status - async def wait( self, timeout: int = 300, @@ -144,7 +143,6 @@ async def wait( # Still in progress (can be "running", "in_progress", "pending", etc.) await asyncio.sleep(poll_interval) - async def fetch(self, format: str = "json") -> Any: """ Fetch job results. @@ -165,7 +163,6 @@ async def fetch(self, format: str = "json") -> Any: self._cached_data = await self._api_client.fetch_result(self.snapshot_id, format=format) return self._cached_data - async def to_result( self, timeout: int = 300, @@ -223,4 +220,3 @@ async def to_result( timing_end=datetime.now(timezone.utc), metadata={"snapshot_id": self.snapshot_id}, ) - diff --git a/src/brightdata/scrapers/linkedin/scraper.py b/src/brightdata/scrapers/linkedin/scraper.py index b1db4c0..3327e67 100644 --- a/src/brightdata/scrapers/linkedin/scraper.py +++ b/src/brightdata/scrapers/linkedin/scraper.py @@ -106,7 +106,6 @@ async def posts( return await self._scrape_urls(url=url, dataset_id=self.DATASET_ID_POSTS, timeout=timeout) - def posts_sync( self, url: Union[str, List[str]], @@ -117,9 +116,11 @@ def posts_sync( See posts() for full documentation. """ + async def _run(): async with self.engine: return await self.posts(url, timeout) + return asyncio.run(_run()) # ============================================================================ @@ -133,7 +134,6 @@ async def posts_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: urls=url, dataset_id=self.DATASET_ID_POSTS, sdk_function=sdk_function or "posts_trigger" ) - def posts_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: """Trigger LinkedIn posts scrape (sync version).""" return asyncio.run(self.posts_trigger(url)) @@ -142,7 +142,6 @@ async def posts_status(self, snapshot_id: str) -> str: """Check LinkedIn posts scrape status.""" return await self._check_status_async(snapshot_id) - def posts_status_sync(self, snapshot_id: str) -> str: """Check LinkedIn posts scrape status (sync version).""" return asyncio.run(self.posts_status(snapshot_id)) @@ -151,7 +150,6 @@ async def posts_fetch(self, snapshot_id: str) -> Any: """Fetch LinkedIn posts scrape results.""" return await self._fetch_results_async(snapshot_id) - def posts_fetch_sync(self, snapshot_id: str) -> Any: """Fetch LinkedIn posts scrape results (sync version).""" return asyncio.run(self.posts_fetch(snapshot_id)) @@ -190,16 +188,17 @@ async def jobs( return await self._scrape_urls(url=url, dataset_id=self.DATASET_ID_JOBS, timeout=timeout) - def jobs_sync( self, url: Union[str, List[str]], timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> Union[ScrapeResult, List[ScrapeResult]]: """Scrape LinkedIn jobs from URLs (sync version).""" + async def _run(): async with self.engine: return await self.jobs(url, timeout) + return asyncio.run(_run()) # ============================================================================ @@ -213,7 +212,6 @@ async def jobs_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: urls=url, dataset_id=self.DATASET_ID_JOBS, sdk_function=sdk_function or "jobs_trigger" ) - def jobs_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: """Trigger LinkedIn jobs scrape (sync version).""" return asyncio.run(self.jobs_trigger(url)) @@ -222,7 +220,6 @@ async def jobs_status(self, snapshot_id: str) -> str: """Check LinkedIn jobs scrape status.""" return await self._check_status_async(snapshot_id) - def jobs_status_sync(self, snapshot_id: str) -> str: """Check LinkedIn jobs scrape status (sync version).""" return asyncio.run(self.jobs_status(snapshot_id)) @@ -231,7 +228,6 @@ async def jobs_fetch(self, snapshot_id: str) -> Any: """Fetch LinkedIn jobs scrape results.""" return await self._fetch_results_async(snapshot_id) - def jobs_fetch_sync(self, snapshot_id: str) -> Any: """Fetch LinkedIn jobs scrape results (sync version).""" return asyncio.run(self.jobs_fetch(snapshot_id)) @@ -270,16 +266,17 @@ async def profiles( return await self._scrape_urls(url=url, dataset_id=self.DATASET_ID, timeout=timeout) - def profiles_sync( self, url: Union[str, List[str]], timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> Union[ScrapeResult, List[ScrapeResult]]: """Scrape LinkedIn profiles from URLs (sync version).""" + async def _run(): async with self.engine: return await self.profiles(url, timeout) + return asyncio.run(_run()) # ============================================================================ @@ -293,7 +290,6 @@ async def profiles_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: urls=url, sdk_function=sdk_function or "profiles_trigger" ) - def profiles_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: """Trigger LinkedIn profiles scrape (sync version).""" return asyncio.run(self.profiles_trigger(url)) @@ -302,7 +298,6 @@ async def profiles_status(self, snapshot_id: str) -> str: """Check LinkedIn profiles scrape status.""" return await self._check_status_async(snapshot_id) - def profiles_status_sync(self, snapshot_id: str) -> str: """Check LinkedIn profiles scrape status (sync version).""" return asyncio.run(self.profiles_status(snapshot_id)) @@ -311,7 +306,6 @@ async def profiles_fetch(self, snapshot_id: str) -> Any: """Fetch LinkedIn profiles scrape results.""" return await self._fetch_results_async(snapshot_id) - def profiles_fetch_sync(self, snapshot_id: str) -> Any: """Fetch LinkedIn profiles scrape results (sync version).""" return asyncio.run(self.profiles_fetch(snapshot_id)) @@ -352,16 +346,17 @@ async def companies( url=url, dataset_id=self.DATASET_ID_COMPANIES, timeout=timeout ) - def companies_sync( self, url: Union[str, List[str]], timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> Union[ScrapeResult, List[ScrapeResult]]: """Scrape LinkedIn companies from URLs (sync version).""" + async def _run(): async with self.engine: return await self.companies(url, timeout) + return asyncio.run(_run()) # ============================================================================ @@ -377,7 +372,6 @@ async def companies_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: sdk_function=sdk_function or "companies_trigger", ) - def companies_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: """Trigger LinkedIn companies scrape (sync version).""" return asyncio.run(self.companies_trigger(url)) @@ -386,7 +380,6 @@ async def companies_status(self, snapshot_id: str) -> str: """Check LinkedIn companies scrape status.""" return await self._check_status_async(snapshot_id) - def companies_status_sync(self, snapshot_id: str) -> str: """Check LinkedIn companies scrape status (sync version).""" return asyncio.run(self.companies_status(snapshot_id)) @@ -395,7 +388,6 @@ async def companies_fetch(self, snapshot_id: str) -> Any: """Fetch LinkedIn companies scrape results.""" return await self._fetch_results_async(snapshot_id) - def companies_fetch_sync(self, snapshot_id: str) -> Any: """Fetch LinkedIn companies scrape results (sync version).""" return asyncio.run(self.companies_fetch(snapshot_id)) diff --git a/src/brightdata/scrapers/linkedin/search.py b/src/brightdata/scrapers/linkedin/search.py index c41d558..8528ac2 100644 --- a/src/brightdata/scrapers/linkedin/search.py +++ b/src/brightdata/scrapers/linkedin/search.py @@ -124,7 +124,6 @@ async def posts( payload=payload, dataset_id=self.DATASET_ID_POSTS, timeout=timeout ) - def posts_sync( self, profile_url: Union[str, List[str]], @@ -137,9 +136,11 @@ def posts_sync( See posts() for documentation. """ + async def _run(): async with self.engine: return await self.posts(profile_url, start_date, end_date, timeout) + return asyncio.run(_run()) # ============================================================================ @@ -187,7 +188,6 @@ async def profiles( payload=payload, dataset_id=self.DATASET_ID_PROFILES, timeout=timeout ) - def profiles_sync( self, firstName: Union[str, List[str]], @@ -199,9 +199,11 @@ def profiles_sync( See profiles() for documentation. """ + async def _run(): async with self.engine: return await self.profiles(firstName, lastName, timeout) + return asyncio.run(_run()) # ============================================================================ @@ -311,7 +313,6 @@ async def jobs( return await self._execute_search(payload=payload, dataset_id=dataset_id, timeout=timeout) - def jobs_sync( self, url: Optional[Union[str, List[str]]] = None, @@ -331,6 +332,7 @@ def jobs_sync( See jobs() for full documentation. """ + async def _run(): async with self.engine: return await self.jobs( @@ -346,6 +348,7 @@ async def _run(): locationRadius=locationRadius, timeout=timeout, ) + return asyncio.run(_run()) # ============================================================================ diff --git a/src/brightdata/sync_client.py b/src/brightdata/sync_client.py index 5a9516e..4a5f91f 100644 --- a/src/brightdata/sync_client.py +++ b/src/brightdata/sync_client.py @@ -96,16 +96,12 @@ def __enter__(self): # Validate token if requested if self._validate_token: - is_valid = self._loop.run_until_complete( - self._async_client.test_connection() - ) + is_valid = self._loop.run_until_complete(self._async_client.test_connection()) if not is_valid: self.__exit__(None, None, None) from .exceptions import AuthenticationError - raise AuthenticationError( - "Token validation failed. Token appears to be invalid." - ) + raise AuthenticationError("Token validation failed. Token appears to be invalid.") return self @@ -116,9 +112,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): try: # Cleanup async client - self._loop.run_until_complete( - self._async_client.__aexit__(exc_type, exc_val, exc_tb) - ) + self._loop.run_until_complete(self._async_client.__aexit__(exc_type, exc_val, exc_tb)) # Give the event loop a moment to process any remaining callbacks # This helps prevent "Unclosed client session" warnings @@ -131,9 +125,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): # Let cancellations propagate if pending: - self._loop.run_until_complete( - asyncio.gather(*pending, return_exceptions=True) - ) + self._loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) except Exception: # Ignore errors during cleanup pass @@ -210,9 +202,7 @@ def token(self) -> str: def __repr__(self) -> str: """String representation.""" - token_preview = ( - f"{self.token[:10]}...{self.token[-5:]}" if self.token else "None" - ) + token_preview = f"{self.token[:10]}...{self.token[-5:]}" if self.token else "None" status = "Initialized" if self._loop else "Not initialized" return f"" @@ -265,7 +255,6 @@ def chatgpt(self) -> "SyncChatGPTScraper": return self._chatgpt - class SyncAmazonScraper: """Sync wrapper for AmazonScraper - COMPLETE with all methods.""" @@ -280,15 +269,11 @@ def products(self, url, **kwargs) -> ScrapeResult: def products_trigger(self, url, **kwargs): """Trigger Amazon products scrape.""" - return self._loop.run_until_complete( - self._async.products_trigger(url, **kwargs) - ) + return self._loop.run_until_complete(self._async.products_trigger(url, **kwargs)) def products_status(self, snapshot_id): """Check Amazon products scrape status.""" - return self._loop.run_until_complete( - self._async.products_status(snapshot_id) - ) + return self._loop.run_until_complete(self._async.products_status(snapshot_id)) def products_fetch(self, snapshot_id): """Fetch Amazon products scrape results.""" @@ -301,9 +286,7 @@ def reviews(self, url, **kwargs) -> ScrapeResult: def reviews_trigger(self, url, **kwargs): """Trigger Amazon reviews scrape.""" - return self._loop.run_until_complete( - self._async.reviews_trigger(url, **kwargs) - ) + return self._loop.run_until_complete(self._async.reviews_trigger(url, **kwargs)) def reviews_status(self, snapshot_id): """Check Amazon reviews scrape status.""" @@ -320,9 +303,7 @@ def sellers(self, url, **kwargs) -> ScrapeResult: def sellers_trigger(self, url, **kwargs): """Trigger Amazon sellers scrape.""" - return self._loop.run_until_complete( - self._async.sellers_trigger(url, **kwargs) - ) + return self._loop.run_until_complete(self._async.sellers_trigger(url, **kwargs)) def sellers_status(self, snapshot_id): """Check Amazon sellers scrape status.""" @@ -406,9 +387,7 @@ def profiles(self, url, **kwargs): return self._loop.run_until_complete(self._async.profiles(url, **kwargs)) def profiles_trigger(self, url, **kwargs): - return self._loop.run_until_complete( - self._async.profiles_trigger(url, **kwargs) - ) + return self._loop.run_until_complete(self._async.profiles_trigger(url, **kwargs)) def profiles_status(self, snapshot_id): return self._loop.run_until_complete(self._async.profiles_status(snapshot_id)) @@ -434,9 +413,7 @@ def comments(self, url, **kwargs): return self._loop.run_until_complete(self._async.comments(url, **kwargs)) def comments_trigger(self, url, **kwargs): - return self._loop.run_until_complete( - self._async.comments_trigger(url, **kwargs) - ) + return self._loop.run_until_complete(self._async.comments_trigger(url, **kwargs)) def comments_status(self, snapshot_id): return self._loop.run_until_complete(self._async.comments_status(snapshot_id)) @@ -467,73 +444,49 @@ def __init__(self, async_scraper, loop): # Posts by profile - NOTE: Must call async methods (not _sync wrappers) because they use asyncio.run() def posts_by_profile(self, url, **kwargs): - return self._loop.run_until_complete( - self._async.posts_by_profile(url, **kwargs) - ) + return self._loop.run_until_complete(self._async.posts_by_profile(url, **kwargs)) def posts_by_profile_trigger(self, url, **kwargs): - return self._loop.run_until_complete( - self._async.posts_by_profile_trigger(url, **kwargs) - ) + return self._loop.run_until_complete(self._async.posts_by_profile_trigger(url, **kwargs)) def posts_by_profile_status(self, snapshot_id): - return self._loop.run_until_complete( - self._async.posts_by_profile_status(snapshot_id) - ) + return self._loop.run_until_complete(self._async.posts_by_profile_status(snapshot_id)) def posts_by_profile_fetch(self, snapshot_id): - return self._loop.run_until_complete( - self._async.posts_by_profile_fetch(snapshot_id) - ) + return self._loop.run_until_complete(self._async.posts_by_profile_fetch(snapshot_id)) # Posts by group def posts_by_group(self, url, **kwargs): - return self._loop.run_until_complete( - self._async.posts_by_group(url, **kwargs) - ) + return self._loop.run_until_complete(self._async.posts_by_group(url, **kwargs)) def posts_by_group_trigger(self, url, **kwargs): - return self._loop.run_until_complete( - self._async.posts_by_group_trigger(url, **kwargs) - ) + return self._loop.run_until_complete(self._async.posts_by_group_trigger(url, **kwargs)) def posts_by_group_status(self, snapshot_id): - return self._loop.run_until_complete( - self._async.posts_by_group_status(snapshot_id) - ) + return self._loop.run_until_complete(self._async.posts_by_group_status(snapshot_id)) def posts_by_group_fetch(self, snapshot_id): - return self._loop.run_until_complete( - self._async.posts_by_group_fetch(snapshot_id) - ) + return self._loop.run_until_complete(self._async.posts_by_group_fetch(snapshot_id)) # Posts by URL def posts_by_url(self, url, **kwargs): return self._loop.run_until_complete(self._async.posts_by_url(url, **kwargs)) def posts_by_url_trigger(self, url, **kwargs): - return self._loop.run_until_complete( - self._async.posts_by_url_trigger(url, **kwargs) - ) + return self._loop.run_until_complete(self._async.posts_by_url_trigger(url, **kwargs)) def posts_by_url_status(self, snapshot_id): - return self._loop.run_until_complete( - self._async.posts_by_url_status(snapshot_id) - ) + return self._loop.run_until_complete(self._async.posts_by_url_status(snapshot_id)) def posts_by_url_fetch(self, snapshot_id): - return self._loop.run_until_complete( - self._async.posts_by_url_fetch(snapshot_id) - ) + return self._loop.run_until_complete(self._async.posts_by_url_fetch(snapshot_id)) # Comments def comments(self, url, **kwargs): return self._loop.run_until_complete(self._async.comments(url, **kwargs)) def comments_trigger(self, url, **kwargs): - return self._loop.run_until_complete( - self._async.comments_trigger(url, **kwargs) - ) + return self._loop.run_until_complete(self._async.comments_trigger(url, **kwargs)) def comments_status(self, snapshot_id): return self._loop.run_until_complete(self._async.comments_status(snapshot_id)) @@ -565,14 +518,10 @@ def __init__(self, async_scraper, loop): # Prompt - Call async methods (prompt) not sync wrappers (prompt_sync) # because sync wrappers use asyncio.run() which conflicts with our persistent loop def prompt(self, prompt_text, **kwargs): - return self._loop.run_until_complete( - self._async.prompt(prompt_text, **kwargs) - ) + return self._loop.run_until_complete(self._async.prompt(prompt_text, **kwargs)) def prompt_trigger(self, prompt_text, **kwargs): - return self._loop.run_until_complete( - self._async.prompt_trigger(prompt_text, **kwargs) - ) + return self._loop.run_until_complete(self._async.prompt_trigger(prompt_text, **kwargs)) def prompt_status(self, snapshot_id): return self._loop.run_until_complete(self._async.prompt_status(snapshot_id)) @@ -585,9 +534,7 @@ def prompts(self, prompts, **kwargs): return self._loop.run_until_complete(self._async.prompts(prompts, **kwargs)) def prompts_trigger(self, prompts, **kwargs): - return self._loop.run_until_complete( - self._async.prompts_trigger(prompts, **kwargs) - ) + return self._loop.run_until_complete(self._async.prompts_trigger(prompts, **kwargs)) def prompts_status(self, snapshot_id): return self._loop.run_until_complete(self._async.prompts_status(snapshot_id)) @@ -641,9 +588,7 @@ def linkedin(self) -> "SyncLinkedInSearchScraper": def instagram(self) -> "SyncInstagramSearchScraper": """Instagram search service.""" if self._instagram is None: - self._instagram = SyncInstagramSearchScraper( - self._async.instagram, self._loop - ) + self._instagram = SyncInstagramSearchScraper(self._async.instagram, self._loop) return self._instagram @property @@ -671,9 +616,7 @@ def __init__(self, async_scraper, loop): self._loop = loop def posts(self, profile_url, **kwargs): - return self._loop.run_until_complete( - self._async.posts(profile_url, **kwargs) - ) + return self._loop.run_until_complete(self._async.posts(profile_url, **kwargs)) def profiles(self, **kwargs): return self._loop.run_until_complete(self._async.profiles(**kwargs)) diff --git a/tests/enes/web_unlocker.py b/tests/enes/web_unlocker.py index a9a72c4..df34ae0 100644 --- a/tests/enes/web_unlocker.py +++ b/tests/enes/web_unlocker.py @@ -33,9 +33,7 @@ async def test_web_unlocker_single_url(): print("šŸ“ URL: https://httpbin.org/html") try: - result = await client.scrape_url( - url="https://httpbin.org/html", response_format="raw" - ) + result = await client.scrape_url(url="https://httpbin.org/html", response_format="raw") print("\nāœ… API call succeeded") print(f"ā±ļø Elapsed: {result.elapsed_ms():.2f}ms" if result.elapsed_ms() else "") @@ -86,9 +84,7 @@ async def test_web_unlocker_json_format(): print("šŸ“ URL: https://httpbin.org/json") try: - result = await client.scrape_url( - url="https://httpbin.org/json", response_format="json" - ) + result = await client.scrape_url(url="https://httpbin.org/json", response_format="json") print("\nāœ… API call succeeded") print(f"ā±ļø Elapsed: {result.elapsed_ms():.2f}ms" if result.elapsed_ms() else "") diff --git a/tests/enes/zones/auto_zones.py b/tests/enes/zones/auto_zones.py index c439793..85470cd 100644 --- a/tests/enes/zones/auto_zones.py +++ b/tests/enes/zones/auto_zones.py @@ -92,9 +92,7 @@ async def attempt_zone_creations(): print(f"\n1ļøāƒ£ Attempting to create Web Unlocker zone: {client.web_unlocker_zone}") try: async with client: - await client.scrape_url( - url="https://example.com", zone=client.web_unlocker_zone - ) + await client.scrape_url(url="https://example.com", zone=client.web_unlocker_zone) print(" āœ… Zone operation completed") results.append(("Web Unlocker", client.web_unlocker_zone, True)) except Exception as e: diff --git a/tests/enes/zones/crud_zones.py b/tests/enes/zones/crud_zones.py index f244f79..b8bdd24 100644 --- a/tests/enes/zones/crud_zones.py +++ b/tests/enes/zones/crud_zones.py @@ -68,9 +68,7 @@ async def test_create_zones(self) -> bool: # Trigger zone creation try: if zone_type == "unblocker": - await temp_client.scrape_url( - url="https://example.com", zone=zone_name - ) + await temp_client.scrape_url(url="https://example.com", zone=zone_name) else: # serp await temp_client.search.google(query="test", zone=zone_name) except Exception: diff --git a/tests/enes/zones/delete_zone.py b/tests/enes/zones/delete_zone.py index 89c7679..67985aa 100644 --- a/tests/enes/zones/delete_zone.py +++ b/tests/enes/zones/delete_zone.py @@ -61,9 +61,7 @@ async def demo_delete_zone(): async with test_client: # Trigger zone creation try: - await test_client.scrape_url( - url="https://example.com", zone=test_zone_name - ) + await test_client.scrape_url(url="https://example.com", zone=test_zone_name) except Exception as e: # Zone might be created even if scrape fails print(f" ā„¹ļø Scrape error (expected): {e}") diff --git a/tests/integration/test_client_integration.py b/tests/integration/test_client_integration.py index 44dc4a7..bf1810f 100644 --- a/tests/integration/test_client_integration.py +++ b/tests/integration/test_client_integration.py @@ -214,9 +214,7 @@ async def test_connection_test_returns_false_on_network_error(self): def test_sync_connection_test_returns_false_on_error(self): """Test sync connection test returns False on errors using SyncBrightDataClient.""" - with SyncBrightDataClient( - token="test_token_123456789", auto_create_zones=False - ) as client: + with SyncBrightDataClient(token="test_token_123456789", auto_create_zones=False) as client: # Should return False, not raise exception is_valid = client.test_connection() assert is_valid is False diff --git a/tests/integration/test_serp_async_mode.py b/tests/integration/test_serp_async_mode.py index d8f3c41..6348649 100644 --- a/tests/integration/test_serp_async_mode.py +++ b/tests/integration/test_serp_async_mode.py @@ -48,9 +48,7 @@ class TestSERPAsyncMode: async def test_google_search_sync_mode_explicit(self, async_client): """Test sync mode still works when explicitly specified.""" result = await async_client.search.google( - query="python programming", - zone=async_client.serp_zone, - mode="sync" # Explicit sync + query="python programming", zone=async_client.serp_zone, mode="sync" # Explicit sync ) assert result.success is True, f"Search failed: {result.error}" @@ -65,7 +63,7 @@ async def test_google_search_default_is_sync(self, async_client): """Test default mode is sync (backwards compatibility).""" result = await async_client.search.google( query="test query", - zone=async_client.serp_zone + zone=async_client.serp_zone, # No mode parameter - should default to sync ) @@ -82,8 +80,8 @@ async def test_google_search_async_mode(self, async_client): query="python programming", zone=async_client.serp_zone, mode="async", - poll_interval=2, # Check every 2 seconds - poll_timeout=30 # Give up after 30 seconds + poll_interval=2, # Check every 2 seconds + poll_timeout=30, # Give up after 30 seconds ) assert result.success is True, f"Async search failed: {result.error}" @@ -101,18 +99,12 @@ async def test_async_mode_returns_same_structure_as_sync(self, async_client): # Run sync mode sync_result = await async_client.search.google( - query=query, - zone=async_client.serp_zone, - mode="sync" + query=query, zone=async_client.serp_zone, mode="sync" ) # Run async mode async_result = await async_client.search.google( - query=query, - zone=async_client.serp_zone, - mode="async", - poll_interval=2, - poll_timeout=30 + query=query, zone=async_client.serp_zone, mode="async", poll_interval=2, poll_timeout=30 ) # Both should succeed @@ -152,7 +144,7 @@ async def test_async_mode_with_short_timeout(self, async_client): zone=async_client.serp_zone, mode="async", poll_interval=1, - poll_timeout=1 # Very short timeout + poll_timeout=1, # Very short timeout ) # Should fail with timeout error @@ -172,7 +164,7 @@ async def test_async_mode_multiple_queries(self, async_client): zone=async_client.serp_zone, mode="async", poll_interval=2, - poll_timeout=60 # Longer timeout for multiple queries + poll_timeout=60, # Longer timeout for multiple queries ) # Should get results for all queries @@ -189,10 +181,7 @@ async def test_async_mode_multiple_queries(self, async_client): async def test_sync_mode_with_location(self, async_client): """Test sync mode with location parameter.""" result = await async_client.search.google( - query="restaurants", - zone=async_client.serp_zone, - location="US", - mode="sync" + query="restaurants", zone=async_client.serp_zone, location="US", mode="sync" ) assert result.success is True @@ -209,7 +198,7 @@ async def test_async_mode_with_location(self, async_client): location="US", mode="async", poll_interval=2, - poll_timeout=30 + poll_timeout=30, ) assert result.success is True @@ -229,7 +218,7 @@ async def test_async_mode_has_timing_metadata(self, async_client): zone=async_client.serp_zone, mode="async", poll_interval=2, - poll_timeout=30 + poll_timeout=30, ) assert result.success is True diff --git a/tests/integration/test_web_unlocker_async_mode.py b/tests/integration/test_web_unlocker_async_mode.py index e05086a..609c0d5 100644 --- a/tests/integration/test_web_unlocker_async_mode.py +++ b/tests/integration/test_web_unlocker_async_mode.py @@ -50,7 +50,7 @@ async def test_scrape_sync_mode_explicit(self, async_client): result = await async_client.scrape_url( url="https://example.com", zone=async_client.web_unlocker_zone, - mode="sync" # Explicit sync + mode="sync", # Explicit sync ) assert result.success is True, f"Scrape failed: {result.error}" @@ -65,7 +65,7 @@ async def test_scrape_default_is_sync(self, async_client): """Test default mode is sync (backwards compatibility).""" result = await async_client.scrape_url( url="https://example.com", - zone=async_client.web_unlocker_zone + zone=async_client.web_unlocker_zone, # No mode parameter - should default to sync ) @@ -82,8 +82,8 @@ async def test_scrape_async_mode(self, async_client): url="https://example.com", zone=async_client.web_unlocker_zone, mode="async", - poll_interval=2, # Check every 2 seconds - poll_timeout=30 # Give up after 30 seconds + poll_interval=2, # Check every 2 seconds + poll_timeout=30, # Give up after 30 seconds ) assert result.success is True, f"Async scrape failed: {result.error}" @@ -101,9 +101,7 @@ async def test_async_mode_returns_same_structure_as_sync(self, async_client): # Run sync mode sync_result = await async_client.scrape_url( - url=url, - zone=async_client.web_unlocker_zone, - mode="sync" + url=url, zone=async_client.web_unlocker_zone, mode="sync" ) # Run async mode @@ -112,7 +110,7 @@ async def test_async_mode_returns_same_structure_as_sync(self, async_client): zone=async_client.web_unlocker_zone, mode="async", poll_interval=2, - poll_timeout=30 + poll_timeout=30, ) # Both should succeed @@ -145,7 +143,7 @@ async def test_async_mode_with_short_timeout(self, async_client): zone=async_client.web_unlocker_zone, mode="async", poll_interval=1, - poll_timeout=1 # Very short timeout + poll_timeout=1, # Very short timeout ) # Should fail with timeout error @@ -158,18 +156,14 @@ async def test_async_mode_with_short_timeout(self, async_client): @pytest.mark.slow async def test_async_mode_multiple_urls(self, async_client): """Test async mode with multiple URLs (batch processing).""" - urls = [ - "https://example.com", - "https://www.example.org", - "https://www.example.net" - ] + urls = ["https://example.com", "https://www.example.org", "https://www.example.net"] results = await async_client.scrape_url( url=urls, zone=async_client.web_unlocker_zone, mode="async", poll_interval=2, - poll_timeout=60 # Longer timeout for multiple URLs + poll_timeout=60, # Longer timeout for multiple URLs ) # Should get results for all URLs @@ -190,7 +184,7 @@ async def test_sync_mode_with_country(self, async_client): url="https://example.com", zone=async_client.web_unlocker_zone, country="US", - mode="sync" + mode="sync", ) assert result.success is True @@ -207,7 +201,7 @@ async def test_async_mode_with_country(self, async_client): country="US", mode="async", poll_interval=2, - poll_timeout=30 + poll_timeout=30, ) assert result.success is True @@ -224,7 +218,7 @@ async def test_async_mode_with_json_response(self, async_client): response_format="json", mode="async", poll_interval=2, - poll_timeout=30 + poll_timeout=30, ) assert result.success is True @@ -247,7 +241,7 @@ async def test_async_mode_has_timing_metadata(self, async_client): zone=async_client.web_unlocker_zone, mode="async", poll_interval=2, - poll_timeout=30 + poll_timeout=30, ) assert result.success is True diff --git a/tests/unit/test_async_unblocker.py b/tests/unit/test_async_unblocker.py index bf504ab..d84e0e7 100644 --- a/tests/unit/test_async_unblocker.py +++ b/tests/unit/test_async_unblocker.py @@ -36,15 +36,10 @@ async def test_trigger_success(self): response.headers.get.return_value = "test_response_id_123" # Mock post_to_url to return async context manager - self.engine.post_to_url = MagicMock( - return_value=MockAsyncContextManager(response) - ) + self.engine.post_to_url = MagicMock(return_value=MockAsyncContextManager(response)) # Trigger request - response_id = await self.client.trigger( - zone="test_zone", - url="https://example.com" - ) + response_id = await self.client.trigger(zone="test_zone", url="https://example.com") # Verify response_id returned assert response_id == "test_response_id_123" @@ -62,16 +57,11 @@ async def test_trigger_with_additional_params(self): response = MagicMock() response.headers.get.return_value = "response_id_456" - self.engine.post_to_url = MagicMock( - return_value=MockAsyncContextManager(response) - ) + self.engine.post_to_url = MagicMock(return_value=MockAsyncContextManager(response)) # Trigger with additional params response_id = await self.client.trigger( - zone="my_zone", - url="https://google.com/search?q=test", - format="raw", - country="US" + zone="my_zone", url="https://google.com/search?q=test", format="raw", country="US" ) assert response_id == "response_id_456" @@ -89,14 +79,9 @@ async def test_trigger_no_response_id(self): response = MagicMock() response.headers.get.return_value = None # No x-response-id - self.engine.post_to_url = MagicMock( - return_value=MockAsyncContextManager(response) - ) + self.engine.post_to_url = MagicMock(return_value=MockAsyncContextManager(response)) - response_id = await self.client.trigger( - zone="test_zone", - url="https://example.com" - ) + response_id = await self.client.trigger(zone="test_zone", url="https://example.com") assert response_id is None @@ -106,14 +91,9 @@ async def test_get_status_ready(self): response = MagicMock() response.status = 200 - self.engine.get_from_url = MagicMock( - return_value=MockAsyncContextManager(response) - ) + self.engine.get_from_url = MagicMock(return_value=MockAsyncContextManager(response)) - status = await self.client.get_status( - zone="test_zone", - response_id="abc123" - ) + status = await self.client.get_status(zone="test_zone", response_id="abc123") assert status == "ready" @@ -129,14 +109,9 @@ async def test_get_status_pending(self): response = MagicMock() response.status = 202 - self.engine.get_from_url = MagicMock( - return_value=MockAsyncContextManager(response) - ) + self.engine.get_from_url = MagicMock(return_value=MockAsyncContextManager(response)) - status = await self.client.get_status( - zone="test_zone", - response_id="xyz789" - ) + status = await self.client.get_status(zone="test_zone", response_id="xyz789") assert status == "pending" @@ -148,37 +123,24 @@ async def test_get_status_error(self): response = MagicMock() response.status = error_code - self.engine.get_from_url = MagicMock( - return_value=MockAsyncContextManager(response) - ) + self.engine.get_from_url = MagicMock(return_value=MockAsyncContextManager(response)) - status = await self.client.get_status( - zone="test_zone", - response_id="err123" - ) + status = await self.client.get_status(zone="test_zone", response_id="err123") assert status == "error", f"Expected 'error' for HTTP {error_code}" @pytest.mark.asyncio async def test_fetch_result_success(self): """Test fetch_result returns parsed JSON for HTTP 200.""" - expected_data = { - "general": {"search_engine": "google"}, - "organic": [{"title": "Result 1"}] - } + expected_data = {"general": {"search_engine": "google"}, "organic": [{"title": "Result 1"}]} response = MagicMock() response.status = 200 response.json = AsyncMock(return_value=expected_data) - self.engine.get_from_url = MagicMock( - return_value=MockAsyncContextManager(response) - ) + self.engine.get_from_url = MagicMock(return_value=MockAsyncContextManager(response)) - data = await self.client.fetch_result( - zone="test_zone", - response_id="fetch123" - ) + data = await self.client.fetch_result(zone="test_zone", response_id="fetch123") assert data == expected_data response.json.assert_called_once() @@ -189,15 +151,10 @@ async def test_fetch_result_not_ready(self): response = MagicMock() response.status = 202 - self.engine.get_from_url = MagicMock( - return_value=MockAsyncContextManager(response) - ) + self.engine.get_from_url = MagicMock(return_value=MockAsyncContextManager(response)) with pytest.raises(APIError) as exc_info: - await self.client.fetch_result( - zone="test_zone", - response_id="pending123" - ) + await self.client.fetch_result(zone="test_zone", response_id="pending123") assert "not ready yet" in str(exc_info.value).lower() assert "202" in str(exc_info.value) @@ -209,15 +166,10 @@ async def test_fetch_result_error(self): response.status = 500 response.text = AsyncMock(return_value="Internal Server Error") - self.engine.get_from_url = MagicMock( - return_value=MockAsyncContextManager(response) - ) + self.engine.get_from_url = MagicMock(return_value=MockAsyncContextManager(response)) with pytest.raises(APIError) as exc_info: - await self.client.fetch_result( - zone="test_zone", - response_id="error123" - ) + await self.client.fetch_result(zone="test_zone", response_id="error123") error_msg = str(exc_info.value) assert "500" in error_msg diff --git a/tests/unit/test_chatgpt.py b/tests/unit/test_chatgpt.py index 20e41df..9fd9e2a 100644 --- a/tests/unit/test_chatgpt.py +++ b/tests/unit/test_chatgpt.py @@ -230,7 +230,9 @@ def test_country_should_be_2_letter_format(self): # Check docstring mentions 2-letter format (async-first API) doc = search.chatGPT.__doc__ - assert doc is not None and ("2-letter" in doc or "2 letter" in doc.replace("-", " ") or "country" in doc.lower()) + assert doc is not None and ( + "2-letter" in doc or "2 letter" in doc.replace("-", " ") or "country" in doc.lower() + ) class TestChatGPTPhilosophicalPrinciples: diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 4f5dc7d..6445f6a 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -71,7 +71,6 @@ def test_client_raises_error_for_non_string_token(self): assert "Invalid token format" in str(exc_info.value) - class TestClientTokenManagement: """Test token management and validation.""" diff --git a/tests/unit/test_scrapers.py b/tests/unit/test_scrapers.py index 49b51b3..4fb7857 100644 --- a/tests/unit/test_scrapers.py +++ b/tests/unit/test_scrapers.py @@ -258,6 +258,7 @@ def test_chatgpt_scraper_has_prompts_method(self): def test_chatgpt_scraper_scrape_raises_not_implemented(self): """Test ChatGPTScraper raises NotImplementedError for scrape().""" import asyncio + scraper = ChatGPTScraper(bearer_token="test_token_123456789") async def test_scrape(): @@ -413,6 +414,7 @@ def test_linkedin_interface_matches_spec(self): def test_chatgpt_interface_matches_spec(self): """Test ChatGPT scraper matches interface specification.""" import asyncio + scraper = ChatGPTScraper(bearer_token="test_token_123456789") # Prompt-based (ChatGPT specific)