o Dhd/@sdZddlZddlmZddlmZddlmZmZm Z ddl m Z ddl m Z mZmZmZmZmZzddlZWn eyAYnwddlmZmZd d lmZmZd d lmZmZmZd d lm Z d d l!m"Z"m#Z#m$Z$e%e&Z'e dddZ(dZ)dZ*dZ+GdddZ,de-de-de ee-ee-ffddZ.de-de ee-ee-ee-ffddZ/de-de-dee fddZ0d e-dee fd!d"Z1de-d#ee-de2fd$d%Z3d&ee-de2fd'd(Z4 )dAde-d*e,d+ee-ddfd,d-Z5d.eed*e,ddfd/d0Z6     dBd1e-d2ee-d3ee d&eee-d4eee-d5ee-de,fd6d7Z7 dCd*e,d8e2de,fd9d:Z8e*e+ddde ddfde-d;e9ded3ee d5ee-de ee-ee-ffd?d@Z:dS)DzC Functions dedicated to website navigation and crawling/spidering. N) ConfigParser)sleep)ListOptionalTuple)RobotFileParser)UrlStore extract_linksfix_relative_urls get_base_urlis_navigation_pageis_not_crawlable)XPathtostring)baselineprune_unwanted_nodes)Responsefetch_response fetch_url)DEFAULT_CONFIG) LANGID_FLAG decode_file load_htmlF) compressedstrictz /robots.txt ic @seZdZdZgdZ   ddedeedeedeeddf d d Zdedefd d Z dedefd dZ de ddfddZ dee ede efddZdedefddZdS)CrawlParametersz6Store necessary information to manage a focused crawl.) startbaselangrulesrefi known_numis_on prune_xpathNrr r!r&returncCsP||_|||_|||_||_|pt|j|_d|_d|_ d|_ ||_ dS)NrT) r _get_base_urlr_get_referencer"r get_rulesr!r#r$r%r&)selfrr r!r&r,S/home/air/segue/gemini/back/venv/lib/python3.10/site-packages/trafilatura/spider.py__init__0s   zCrawlParameters.__init__cCst|}|s td||S)z#Set reference domain for the crawl.zcannot start crawl: )r ValueError)r+rrr,r,r-r(AszCrawlParameters._get_base_urlcCs"|ddkr|dddS|S)zDetermine the reference URL./rr)countrsplit)r+rr,r,r-r)Hs"zCrawlParameters._get_reference url_storecCs(t||j|_t||j|_dS)z*Adjust crawl data based on URL store info.N)boolfind_unvisited_urlsrr%lenfind_known_urlsr$)r+r4r,r,r-update_metadataLszCrawlParameters.update_metadatatodocs|sgSfdd|DS)z.Prepare the todo list, excluding invalid URLs.cs$g|]}|jkrj|vr|qSr,)rr").0ur+r,r- Us$z/CrawlParameters.filter_list..r,)r+r:r,r=r- filter_listQszCrawlParameters.filter_listlinkcCs*|j s |jd|o|j|vot| S)z9Run checks: robots.txt rules, URL type and crawl breadth.*)r! can_fetchr"r )r+r@r,r,r- is_valid_linkWs zCrawlParameters.is_valid_linkNNN)__name__ __module__ __qualname____doc__ __slots__strrrr.r(r)rr9rr?r5rCr,r,r,r-r,s, r htmlstringhomepager'cCsd|vr d|vr ||fSt|}|dur||fS|d}|r#|dnd}|r+d|vr5td|||fS|dd d d}|d sRt |}t ||}t |}|durbt d |d Std|||fS)z:Check if there could be a redirection by meta-refresh tag.z "refresh"z "REFRESH"Nz@.//meta[@http-equiv="refresh" or @http-equiv="REFRESH"]/@contentr;zno redirect found: %srzurl=httpzfailed redirect: %s)NNzsuccessful redirect: %s) rxpathlogginginfosplitstriplowerreplace startswithr r rwarning)rKrL html_treeresultsresulturl2base_url newhtmlstringr,r,r-refresh_detection`s,      r_cCszt|dd}|r |js dS|j|dfvrtd|j|j}t|j}t||\}}|dur0dStd|||t|fS)zBCheck if the homepage is redirected and return appropriate values.FdecoderDr0zfollowed homepage redirect: %sNzfetching homepage OK: %s) rdataurlrQrRrr_debugr )rLresponserKnew_htmlstring new_homepager,r,r-probe_alternative_homepages    rh robots_urlrbc CsXt}||z ||W|Sty+}z td|WYd}~dSd}~ww)zEParse a robots.txt file with the standard library urllib.robotparser.zcannot read robots.txt: %sN)rset_urlparse splitlines ExceptionLOGGERerror)rirbr!excr,r,r- parse_robotss  rqr]cCs"|t}t|}|rt||SdS)z?Attempt to fetch and parse robots.txt file for a given website.N)ROBOTS_TXT_URLrrq)r]rirbr,r,r-r*sr*languagecCs8|r|rtrt|\}}}t|\}}t||kSdS)zRun a baseline extraction and use a language detector to check if the content matches the target language. Return True if language checks are bypassed.T)rr py3langidclassifyr5)rKrs_textr[r,r,r-is_target_languages  rxr:cCstdd|DS)z6Probe if there are still navigation URLs in the queue.css|]}t|VqdS)N)r )r;rcr,r,r- sz&is_still_navigation..)any)r:r,r,r-is_still_navigationsr{rMparamsrccCst||jsdS|r3|jdur3t|jtr|jg|_t|}|dur3t|dd|jD}t|}gg}}t ||p>|j d|jdddD]}| |sNqFt |rX| |qF| |qFtj||ddS)zExamine the HTML code and process the retrieved internal links. Extract and filter new internal links after an optional language check. Store the links in todo-list while prioritizing the navigation ones.NcSsg|]}t|qSr,)r)r;xr,r,r-r>sz!process_links..FT) pagecontentrc external_boolrswith_navr)urls appendleft)rxr r& isinstancerJrrrrar rrCr append URL_STOREadd_urls)rKr|rctreelinkslinks_priorityr@r,r,r- process_linkss0        rrecCs<|dus|js dStj|jgddtt|j||jdS)z2Convert urllib3 response object and extract links.NT)visited)rbrrrcrrr)rer|r,r,r-process_responsesrrr r!knownr&cCsvt||||}tj|p gddtj||dt|j|j|s4tj|jgddt|dd}|S| t|S)zInitialize crawl by setting variables, copying values to the URL store and retrieving the initial page if the crawl starts.T)rr)rF)initial) rrrr? store_rulesrr!r crawl_pager9)rr r!r:rr&r|r,r,r- init_crawls   rrcCst|j}|sd|_tt|j|_|S|jd7_|r:t|\}}}|r9|r9|r9t |gt |||dn t |dd}t ||| t|S)z6Examine a webpage, extract navigation links and links.Fr)rcr`)rget_urlrr%r7r8r$r#rhrrrrr9)r|rrcrKrL new_base_urlrer,r,r-rs      r max_seen_urlsmax_known_urls known_linksconfigc Cst||||||} tj| j|ddd} | jr7| j|kr7| j|kr7t| } t | | jr7| j|kr7| j|ks"t t t | j}t t t| j}||fS)aBasic crawler targeting pages of interest within a website. Args: homepage: URL of the page to first page to fetch, preferably the homepage of a website. max_seen_urls: maximum number of pages to visit, stop iterations at this number or at the exhaustion of pages on the website, whichever comes first. max_known_urls: stop if the total number of pages "known" exceeds this number. todo: provide a previously generated list of pages to visit / crawl frontier. known_links: provide a list of previously known pages. lang: try to target links according to language heuristics. config: use a different configuration (configparser format). rules: provide politeness rules (urllib.robotparser.RobotFileParser() format). prune_xpath: remove unwanted elements from the HTML pages using XPath. Returns: List of pages to visit, deque format, possibly empty if there are no further pages to visit. Set of known links. DEFAULT SLEEP_TIME)default)rrget_crawl_delayrgetfloatr%r#r$rrlistdictfromkeysr6r8) rLrrr:rr rr!r&r| sleep_timer,r,r-focused_crawler2sr)rM)NNNNN)F);rHrQ configparserrtimertypingrrrurllib.robotparserrcourlanrr r r r r rt ImportError lxml.etreerrcorerr downloadsrrrsettingsrutilsrrr getLoggerrErnrrr MAX_SEEN_URLSMAX_KNOWN_URLSrrJr_rhrqr*r5rxr{rrrrintrr,r,r,r-s         4 &   &     !