a
    bg                     @   sH   d dl mZmZmZ d dlZd dlmZ d dlmZ G dd deZ	dS )    )IteratorListUnionN)Document)
BaseLoaderc                   @   sB   e Zd ZdZd
eeeee f edddZe	e
 dddZd	S )BrowserlessLoaderz3Load webpages with `Browserless` /content endpoint.T	api_tokenurlstext_contentc                 C   s   || _ || _|| _dS )z0Initialize with API token and the URLs to scrapeNr   )selfr	   r
   r    r   ~/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langchain_community/document_loaders/browserless.py__init__   s    zBrowserlessLoader.__init__)returnc                 c   s   | j D ]}| jr^tjdd| ji|ddigdd}t| d d d	 d d
 d|idV  qtjdd| jid|id}t|jd|idV  qdS )zLazy load Documents from URLs.z$https://chrome.browserless.io/scrapetokenselectorbody)urlelements)paramsjsondatar   resultstextsource)Zpage_contentmetadataz%https://chrome.browserless.io/contentr   N)r
   r   requestspostr	   r   r   r   )r   r   responser   r   r   	lazy_load   s:    


zBrowserlessLoader.lazy_loadN)T)__name__
__module____qualname____doc__strr   r   boolr   r   r   r    r   r   r   r   r   	   s    
r   )
typingr   r   r   r   Zlangchain_core.documentsr   Z)langchain_community.document_loaders.baser   r   r   r   r   r   <module>   s   