a ag>@sUddlZddlmZddlmZmZmZmZmZm Z m Z m Z m Z m Z erbddlmZmZmZmZhdZe eed<Gddde ZGd d d ZdS) N)platform) TYPE_CHECKINGAnyDictIterableListOptionalSetTuple TypedDictUnion)Browser CDPSessionPagesync_playwright> htmlmetabrZiframebodystylepathheadtitlescriptz::markersvgblack_listed_elementsc@sneZdZUdZeed<eed<eeed<eeed<eeed<e ed<eed<eed <eed <eed <d S) ElementInViewPortzIA typed dictionary containing information about elements in the viewport. node_indexbackend_node_id node_name node_value node_meta is_clickableorigin_xorigin_ycenter_xcenter_yN) __name__ __module__ __qualname____doc__str__annotations__intrrboolr/r/m/var/www/html/cobodadashboardai.evdpl.com/venv/lib/python3.9/site-packages/langchain/chains/natbot/crawler.pyr$s    rc@seZdZdZddddZeddddZedd d d Zeee fdd d dZ eee feddddZ ddddZ e edddZdS)Crawlera A crawler for web pages. **Security Note**: This is an implementation of a crawler that uses a browser via Playwright. This crawler can be used to load arbitrary webpages INCLUDING content from the local file system. Control access to who can submit crawling requests and what network access the crawler has. Make sure to scope permissions to the minimal permissions necessary for the application. See https://python.langchain.com/docs/security for more information. N)returncCslzddlm}Wnty*tdYn0|jjdd|_|j|_|j ddd||dS) Nr)rz\Could not import playwright python package. Please install it with `pip install playwright`.F)Zheadlessii8)widthheight) playwright.sync_apir ImportErrorstartZchromiumZlaunchZbrowserZnew_pagepageZset_viewport_size)selfrr/r/r0__init__Es   zCrawler.__init__)urlr2cCs:|jjd|vr|nd|d|jj|j|_i|_dS)Nz://zhttp://)r;)r8ZgotocontextZnew_cdp_sessionclientpage_element_buffer)r9r;r/r/r0 go_to_pageUszCrawler.go_to_page) directionr2cCs.|dkr|jdn|dkr*|jddS)Nupz(document.scrollingElement || document.body).scrollTop = (document.scrollingElement || document.body).scrollTop - window.innerHeight;Zdownz(document.scrollingElement || document.body).scrollTop = (document.scrollingElement || document.body).scrollTop + window.innerHeight;)r8evaluate)r9r@r/r/r0scrollZszCrawler.scroll)idr2cCsRd}|j||jt|}|rF|d}|d}|jj||ntddS)Nz links = document.getElementsByTagName("a"); for (var i = 0; i < links.length; i++) { links[i].removeAttribute("target"); } r%r&zCould not find element)r8rBr>getr-mouseclickprint)r9rDjselementxyr/r/r0rGds z Crawler.click)rDtextr2cCs|||jj|dS)N)rGr8keyboardtype)r9rDrMr/r/r0rOws z Crawler.typecCs|jjddS)NZEnter)r8rNZpress)r9r/r/r0enter{sz Crawler.entercKs|j}|j}t}g}|d}tdkr6|dkr6d}|d}|d}|d}|d} ||} || } d} d} |d d d t| dt| d |j d gd d d}|d|dd }|d}|d}|d}|d}|d|dt |dd}|d}|d}|d}|d}|d}|d}d }i}g}ddi}ddi}t t t t t d d!d"} tttftt tt t fd#fd$d% }!tt tt t tfft tt t ttt t tfd&fd'd( tD]\}"}#|"}$|#}%|d)|"|%|$\}&}'|d*|"|%|$\}(})z||"}WnYqYn0|%tvrXq||\}*}+},}-|*|}*|+|}+|,|},|-|}-|*}.|+}/|*|,}0|+|-}1|.| ko|0|ko|/| ko|1|k}2|2sАqg}3|!||"gd+}4|&p|(}5|5sdn|&rt |'nt |)}6|5sdn|t |6g}7|%d,krv|5rv|7rv||"}8|8d-ks|8d.krdq|7d/|8d0nr|%d1kr|4d/d2ks|%d*krd*}%|4d/d|4D]8}9|5r|7r|7d3|9|4|9d4n|3|4|9qd}:||"d kr||"}:|:d-krbqnL|%d1krb|"|vrb|:durb||"};||;}<|;d krb||D]}?|?d6}@|?d7}%|?d8}:|?d9}A|?d:}B|:r$|:d;nd<}Cd<}D|@|vr||@D]Z}E|Ed/}F|Ed}G|Fd3kr|Br|Ed=}H|B|Hd>|Gd?n|C|Gd;7}Cq>|Brd;|B}Id;|I}D|Cd<|Cd|DdE|CdF|JdE n|=dC|JdD|>|DdG|>d7}>qtdHt||=S)INzwindow.devicePixelRatiodarwinzwindow.pageYOffsetzwindow.pageXOffsetzwindow.screen.widthzwindow.screen.heightrz[scrollbar {:0.2f}-{:0.2f}%])rKrLrMzDOMSnapshot.captureSnapshotT)ZcomputedStylesZincludeDOMRectsZincludePaintOrderstringsZ documentsnodesZ backendNodeId attributesZ nodeValueZ parentIndexZnodeNameZ isClickableindexZ inputValuevaluelayoutZ nodeIndexboundsz-1FN)rhas_click_handlerr2cSs<|dkr dS|dkrdS|dkr$dS|dks0|r4dSdSdS)NalinkinputimgbuttonrMr/)rr\r/r/r0 convert_namesz#Crawler.crawl..convert_name)rVkeysr2csdi}tt|fdD]H\}}|dkr(q|}|}||vr|||<|||s|Sq|S)NrSr)zipiterremove)rVrcvaluesZ key_indexZ value_indexkeyrX)rTr/r0find_attributess  z&Crawler.crawl..find_attributes) hash_treetagnode_idr parent_idr2c sxt|}||vr8|}|}|||||||\}} ||krVd|f} n|rdd| f} nd} | |t|<| S)NTr[)r+lower) rjrkrlrrmZ parent_id_str parent_nameZgrand_parent_idZis_parent_desc_anchor anchor_idrXadd_to_hash_treeZ node_namesparentrTr/r0rrs      z'Crawler.crawl..add_to_hash_treer]ra)rO placeholderz aria-labelrZaltz#text|u•rO)rOrXr_submit attribute)rOrhrX) rrrr r!r"r#r$r%r&rrr r"r! rhz=""r^r`ZtextareazzParsing time: {:0.2f} seconds)r8r>timerBrappendformatroundr=sendsetrr+r.rr-rrr enumeraternrWr setdefaultrEpopjoinstriprH)Kr9r8r>r7Zpage_state_as_textZdevice_pixel_ratioZwin_upper_boundZwin_left_boundZ win_widthZ win_heightZwin_right_boundZwin_lower_boundZpercentage_progress_startZpercentage_progress_endtreeZdocumentrUrrVr r"Z input_valueZinput_value_indexZinput_value_valuesrYZlayout_node_indexrZcursorZ child_nodesZelements_in_view_portZanchor_ancestryZbutton_ancestryrbrirWZnode_name_indexZ node_parentrZis_ancestor_of_anchorrpZis_ancestor_of_buttonZ button_idrKrLr3r4Zelem_left_boundZelem_top_boundZelem_right_boundZelem_lower_boundZpartially_is_in_viewportZ meta_dataZelement_attributesZancestor_exceptionZancestor_node_keyZ ancestor_noderMrhZelement_node_valueZnode_input_text_indexZ text_indexZelements_of_interestZ id_counterrJrZnode_is_clickableZnode_meta_dataZ inner_textrchildZ entry_typeZ entry_valueZ entry_keyZ meta_stringZconverted_node_namer/rqr0crawl~s           #                                   z Crawler.crawl)r'r(r)r*r:r+r?rCr r-rGrOrPrrr/r/r/r0r13s r1)r}sysrtypingrrrrrrr r r r r5r rrrrr+r,rr1r/r/r/r0s  0