a
    bg/"                     @   s   d dl Z d dlmZ d dlmZ d dlmZmZmZmZm	Z	m
Z
mZmZ d dlZd dlmZ d dlmZ d dlmZmZmZ d dlmZ G d	d
 d
eeeZG dd deZG dd deeZdS )    N)ABC)datetime)AnyCallableDictIteratorListLiteralOptionalUnionDocumentget_from_dict_or_env)	BaseModelfield_validatormodel_validator)
BaseLoaderc                   @   sf   e Zd ZU dZeed< eed< dZeed< eddee	e
dd	d
Zee	eef dddZdS )BaseGitHubLoaderz Load `GitHub` repository Issues.repoaccess_tokenzhttps://api.github.comgithub_api_urlbefore)mode)valuesreturnc                 C   s   t |dd|d< |S )z1Validate that access token exists in environment.r   ZGITHUB_PERSONAL_ACCESS_TOKENr   )clsr    r   y/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langchain_community/document_loaders/github.pyvalidate_environment   s    z%BaseGitHubLoader.validate_environmentr   c                 C   s   dd| j  dS )Nzapplication/vnd.github+jsonzBearer )AcceptAuthorization)r   selfr   r   r   headers!   s    
zBaseGitHubLoader.headersN)__name__
__module____qualname____doc__str__annotations__r   r   classmethodr   r   r   propertyr%   r   r   r   r   r      s   
r   c                   @   sR  e Zd ZU dZdZeed< dZee	e
d df ed< dZee
d  ed< dZee ed	< dZee ed
< dZee ed< dZeee  ed< dZee
d  ed< dZee
d  ed< dZee ed< dZee	 ed< dZee	 ed< edeee ee dddZee dddZeedddZe edddZ!e eddd Z"dS )!GitHubIssuesLoaderz#Load issues of a GitHub repository.Tinclude_prsN)*none	milestone)openclosedallstateassigneecreator	mentionedlabels)createdupdatedcommentssort)ascdesc	directionsincepageper_page)vr   c                 C   s:   |r6zt |d W n  ty4   td| Y n0 |S )Nz%Y-%m-%dT%H:%M:%SZz\Invalid value for 'since'. Expected a date string in YYYY-MM-DDTHH:MM:SSZ format. Received: )r   strptime
ValueError)r   rE   r   r   r   validate_sinceL   s    
z!GitHubIssuesLoader.validate_sincer    c                 c   s   | j }|rtj|| jd}|  | }|D ]&}| |}| jsN|jd rNq.|V  q.|j	r|j	dr| j
s| js|j	d d }qd}qdS )a  
        Get issues of a GitHub repository.

        Returns:
            A list of Documents with attributes:
                - page_content
                - metadata
                    - url
                    - title
                    - creator
                    - created_at
                    - last_update_time
                    - closed_time
                    - number of comments
                    - state
                    - labels
                    - assignee
                    - assignees
                    - milestone
                    - locked
                    - number
                    - is_pull_request
        r%   is_pull_requestnexturlN)rL   requestsgetr%   raise_for_statusjsonparse_issuer/   metadatalinksrC   rD   )r$   rL   responseZissuesissuedocr   r   r   	lazy_loadY   s(    

zGitHubIssuesLoader.lazy_load)rU   r   c                 C   s   |d |d |d d |d |d |d dd	 |d
 D |d rL|d d nd|d rb|d d nd|d |d d|v d}|d dur|d nd}t ||dS )z5Create Document objects from a list of GitHub issues.Zhtml_urltitleuserlogin
created_atr=   r6   c                 S   s   g | ]}|d  qS )namer   ).0labelr   r   r   
<listcomp>       z2GitHubIssuesLoader.parse_issue.<locals>.<listcomp>r:   r7   Nr2   lockednumberZpull_request)rL   rX   r8   r[   r=   r6   r:   r7   r2   ra   rb   rJ   body Zpage_contentrR   r   )r$   rU   rR   contentr   r   r   rQ      s    
zGitHubIssuesLoader.parse_issuec                 C   sh   | j rd| j n| j }| j| j| j| j| j|| j| j| j	| j
| jd}dd | D }d|}|S )z'Create query parameters for GitHub API.,)r2   r6   r7   r8   r9   r:   r>   rA   rB   rC   rD   c                 S   s&   g | ]\}}|d ur| d| qS )N=r   )r]   krE   r   r   r   r_      s   z3GitHubIssuesLoader.query_params.<locals>.<listcomp>&)r:   joinr2   r6   r7   r8   r9   r>   rA   rB   rC   rD   items)r$   r:   Zquery_params_dictZquery_params_listquery_paramsr   r   r   rm      s$    
zGitHubIssuesLoader.query_paramsc                 C   s   | j  d| j d| j S )zCreate URL for GitHub API./repos/z/issues?)r   r   rm   r#   r   r   r   rL      s    zGitHubIssuesLoader.url)#r&   r'   r(   r)   r/   boolr+   r2   r   intr	   r6   r
   r7   r*   r8   r9   r:   r   r>   rA   rB   rC   rD   r   r,   rH   r   r   rW   dictrQ   r-   rm   rL   r   r   r   r   r.   )   s,   
+r.   c                   @   sh   e Zd ZU dZdZeed< eeege	f  ed< e
e dddZeedd	d
Zee dddZdS )GithubFileLoaderzLoad GitHub Filemainbranchfile_filterr    c                    sR    j  d j d j d}tj| jd}|  | d } fdd|D S )Nrn   z/git/trees/z?recursive=1rI   treec                    s$   g | ]} j r  |d  r|qS )path)ru   )r]   fr#   r   r   r_      s   z3GithubFileLoader.get_file_paths.<locals>.<listcomp>)r   r   rt   rM   rN   r%   rO   rP   )r$   base_urlrT   	all_filesr   r#   r   get_file_paths   s    

zGithubFileLoader.get_file_paths)rw   r   c                 C   sx   | j rd| j  nd}| j d| j d| | }tj|| jd}|  t| t	rt| d }t
|dS dS )Nz?ref=rd   rn   z
/contents/rI   rf   zutf-8)rt   r   r   rM   rN   r%   rO   
isinstancerP   rq   base64	b64decodedecode)r$   rw   Zqueryparamsry   rT   Zcontent_encodedr   r   r   get_file_content_by_path   s    z)GithubFileLoader.get_file_content_by_pathc                 c   sz   |   }|D ]h}| |d }|dkr(q|d |d | j d| j d|d  d| j d|d  	d}t||dV  qd S )Nrw   rd   sha/type)rw   r   sourcere   )r{   r   r   r   rt   r   )r$   filesfilerf   rR   r   r   r   rW      s    zGithubFileLoader.lazy_loadN)r&   r'   r(   r)   rt   r*   r+   r
   r   ro   r   r   r{   r   r   r   rW   r   r   r   r   rr      s   
rr   )r}   abcr   r   typingr   r   r   r   r   r	   r
   r   rM   Zlangchain_core.documentsr   Zlangchain_core.utilsr   Zpydanticr   r   r   Z)langchain_community.document_loaders.baser   r   r.   rr   r   r   r   r   <module>   s   ( 