Remove all not working provider (#1679)

Fix many providers
Add selenium-wire to requierments
This commit is contained in:
H Lohaus 2024-03-12 02:06:06 +01:00 committed by GitHub
parent 843f6db564
commit 6ef282de3a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
57 changed files with 696 additions and 542 deletions

View file

@ -1,16 +1,20 @@
from __future__ import annotations
from aiohttp import ClientSession, ClientResponse, ClientTimeout
from typing import AsyncGenerator, Any
from aiohttp import ClientSession, ClientResponse, ClientTimeout, BaseConnector
from typing import AsyncIterator, Any, Optional
from ..providers.helper import get_connector
from .defaults import DEFAULT_HEADERS
from ..errors import MissingRequirementsError
class StreamResponse(ClientResponse):
async def iter_lines(self) -> AsyncGenerator[bytes, None]:
async def iter_lines(self) -> AsyncIterator[bytes]:
async for line in self.content:
yield line.rstrip(b"\r\n")
async def iter_content(self) -> AsyncIterator[bytes]:
async for chunk in self.content.iter_any():
yield chunk
async def json(self) -> Any:
return await super().json(content_type=None)
@ -27,4 +31,16 @@ class StreamSession(ClientSession):
response_class=StreamResponse,
connector=get_connector(kwargs.get("connector"), proxies.get("https")),
headers=headers
)
)
def get_connector(connector: BaseConnector = None, proxy: str = None, rdns: bool = False) -> Optional[BaseConnector]:
if proxy and not connector:
try:
from aiohttp_socks import ProxyConnector
if proxy.startswith("socks5h://"):
proxy = proxy.replace("socks5h://", "socks5://")
rdns = True
connector = ProxyConnector.from_url(proxy, rdns=rdns)
except ImportError:
raise MissingRequirementsError('Install "aiohttp_socks" package for proxy support')
return connector