o DŒhCã @s¤dZddlZddlZddlmZmZddlmZmZm Z m Z m Z ddl m Z ddlmZmZmZddlmZe e¡Ze d ¡Ze d ¡Ze d ¡Ze d ¡Ze d ¡Ze d¡Ze d¡Ze d¡Ze d¡Z e d¡Z!d-de"dee"dee"fdd„Z#de"de"fdd„Z$ d.de"de%dee"de"fdd„Z&de"de"fd d!„Z'd"e"de"fd#d$„Z(d-d%e"dee"de"fd&d'„Z)   (d/d)ee e"fde%dee"d*e%de"f d+d,„Z*dS)0z0 Functions performing URL trimming and cleaning éN)ÚOptionalÚUnion)Úparse_qsÚquoteÚ urlencodeÚ urlunsplitÚ SplitResulté)Ú is_valid_url)ÚALLOWED_PARAMSÚ LANG_PARAMSÚ TARGET_LANGS)Ú_parsez https?://zZ(https?://[^">&? ]+?)(?:https?://)|(?:https?://[^/]+?/[^/]+?[&?]u(rl)?=)(https?://[^"> ]+)z)https?://.+?(https?://.+?)(?:https?://|$)z(?<=\w):(?:80|443)z/+z^(?:/\.\.(?![^/]))+z|{.+?}z/\&$z (.*?)[<>"\s]zí^(?:dc|fbc|gc|twc|yc|ysc)lid|^(?:click|gbra|msclk|igsh|partner|wbra)id|^(?:ads?|mc|ga|gs|itm|mc|mkt|ml|mtm|oly|pk|utm|vero)_|(?:\b|_)(?:aff|affi|affiliate|campaign|cl?id|eid|ga|gl|kwd|keyword|medium|ref|referr?er|session|source|uid|xtor)ÚurlÚlanguageÚreturnc Cs,z tt|ƒd|ƒWSttfyYdSw)z4Helper function: chained scrubbing and normalizationFN)Ú normalize_urlÚ scrub_urlÚAttributeErrorÚ ValueError)rr©rúN/home/air/segue/gemini/back/venv/lib/python3.10/site-packages/courlan/clean.pyÚ clean_url0s ÿrcCsJd | ¡¡ d¡}| d¡r| dd¡ dd¡}t d|¡}t d| dd¡¡}t  |¡}t |ƒdkrpd|vrpt   d t |ƒ|¡t  |¡}|rYt|dƒrY|d}t   d |¡nt |¡}|rpt|dƒrp|d}t   d |¡t |¡}|r{|d}t |ƒd krt   d |d d…dt |ƒ¡| d¡dksž| d¡dkr£| d¡}|S)z@Strip unnecessary parts and make sure only one URL is consideredÚz  z z&ú&r zweb.archive.orgzdouble url: %s %sztaking url: %siôz$invalid-looking link %s of length %dNé2u…ú/éz://)ÚjoinÚsplitÚstripÚ startswithÚreplaceÚREMAINING_MARKUPÚsubÚ TRAILING_AMPÚ PROTOCOLSÚfindallÚlenÚLOGGERÚdebugÚ SELECTIONÚmatchr Ú MIDDLE_URLÚTRAILING_PARTSÚcountÚrstrip)rÚ protocolsr,rrrr8s4ÿ         rFÚ querystringÚstrictcCs |sdSt|ƒ}i}t|ƒD];}| ¡}|r |tvr|tvrqnt |¡r&q|tvrC|tvrCt||dƒt|vrCt   d||¡t ‚||||<qt |ddS)zStrip unwanted query elementsrrzbad lang: %s %sT)Údoseq) rÚsortedÚlowerr r Ú TRACKERS_REÚsearchr Ústrr)r*rr)r2r3rÚqdictÚnewqdictÚqelemÚteststrrrrÚ clean_queryjs& €  r>Ústringc Csvd|vr|Sg}| d¡D](}| ¡ d¡r0z | d¡ d¡}Wnty/t d|¡Ynw| |¡q d  |¡S)z@Probe for punycode in lower-cased hostname and try to decode it.zxn--Ú.Úutf8Úidnazinvalid utf/idna string: %s) rr6r!ÚencodeÚdecodeÚ UnicodeErrorr)r*Úappendr)r?ÚpartsÚpartrrrÚdecode_punycode‹s ÿ  rIÚurl_partcCs t|ddS)zbNormalize URLs parts (specifically path and fragment) while accounting for certain characters.z/%!=:,-)Úsafe)r)rJrrrÚnormalize_parts rLÚfragmentcCs:d|vrd|vrt|d|ƒ}t|ƒSt |¡rd}t|ƒS)zNLook for trackers in URL fragments using query analysis, normalize the output.ú=rFr)r>r7r8rL)rMrrrrÚnormalize_fragment£s  þrOTÚ parsed_urlÚtrailing_slashc CsÖt|ƒ}|j ¡}t|j ¡ƒ}z |jdvrt d|¡}Wn ty&Ynwt t  dt  d|j ¡¡ƒ}t |j||ƒpÚqueryr(Úendswithr0rOrMr) rPr3rrQrSrTÚnewpathÚnewqueryÚ newfragmentrrrr­s0   € ÿÿþ ÿ r)N)FN)FNT)+Ú__doc__ÚloggingÚreÚtypingrrÚ urllib.parserrrrrÚfiltersr Úsettingsr r r ÚurlutilsrÚ getLoggerÚ__name__r)Úcompiler&r+r-rVrXrWr#r%r.r7r9rrÚboolr>rIrLrOrrrrrÚsh    ÿ       ÿ 3ÿÿÿÿ þ! ü ÿþýüû