a
    bgN                     @   sP   d dl Z d dlmZ d dlmZmZ d dlmZ d dlm	Z	 G dd de	Z
dS )    N)Path)ListUnion)Document)
BaseLoaderc                   @   s8   e Zd ZdZeeef dddZee	 dddZ
dS )	CoNLLULoaderzLoad `CoNLL-U` files.	file_pathc                 C   s
   || _ dS )zInitialize with a file path.Nr   )selfr	    r   y/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langchain_community/document_loaders/conllu.py__init__   s    zCoNLLULoader.__init__)returnc                 C   s   t | jdd0}ttj|dd}dd |D }W d   n1 sD0    Y  d}t|D ]B\}}|d	 d
ks~|t|d kr||d 7 }qZ||d d 7 }qZdt| ji}t||dgS )zLoad from a file path.utf8)encoding	)	delimiterc                 S   s   g | ]}t |d kr|qS )   )len).0liner   r   r   
<listcomp>       z%CoNLLULoader.load.<locals>.<listcomp>N 	   zSpaceAfter=Nor    source)Zpage_contentmetadata)	openr	   listcsvreader	enumerater   strr   )r
   fZtsvlinestextir   r   r   r   r   load   s    ,zCoNLLULoader.loadN)__name__
__module____qualname____doc__r   r#   r   r   r   r   r(   r   r   r   r   r   
   s   r   )r    pathlibr   typingr   r   Zlangchain_core.documentsr   Z)langchain_community.document_loaders.baser   r   r   r   r   r   <module>   s
   