x[hddlZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z m Z ddl m Z ddlmZddlmZddlmZddlmZddlmZmZmZmZmZmZmZmZmZdd l m!Z!m"Z"m#Z#m$Z$ddl%Z%dd l%m&Z&dd l'm(Z(m)Z)m*Z*ejVe,Z-d Z.eed ge/fZ0dZ1dZ2de3fdZ4dKde5de6de7ddfdZ8de5ddfdZ9dedfdZ:GddZ;Gdde;Z<Gdde;Z=Gd d!Z>Gd"d e?Z@d#ZAd$e5de6fd%ZBdd&d'e@d(e0dee6fd)ZCdddd*ddddd+d+ddd,d+d-d,d.d(e0d/e/de>fd0ZD dLd1ed2efd3e5de3d4ejd5ee6def d6ZF dMd1ed2efd8ee5d9e6de3deee5ee>ff d:ZGGd;d d?e6dee6d@edAeedBee6d(e0dCeeee6ge6fdDe5dEe/dFe6fdGZJGdHdIZK dLdJZLy)NN)ThreadPoolExecutor TimeoutError as_completed) parsedate)partial) NOT_FOUND)count)create_default_context) AnyCallableIteratorListMapping NamedTupleOptionalTupleUnion)quoteurlparseurlsplit urlunparse) exceptions) performanceutilversionREDACTEDUrlErrorcrtt|d}|ds|dr |d|d<d|d<t|S)Nhttp)scheme)listrr)url parsed_urls 6/usr/lib/python3/dist-packages/cloudinit/url_helper.py _cleanurlr(1sChs623J a=Z]#1 1  1 j !!c0d}|}|D] }|||} |S)Nctt|}|d}|r|jds|dz }|tt |dz }||d<t |S)Nr"/z/:)safe)r$rendswithrstrr)r%add_on url_parsedpaths r'combine_singlez#combine_url..combine_single<sY(3-( !}  c* CKD c&k-- 1 *%%r))baseadd_onsr3r%r0s r' combine_urlr7;s.& C*S&)* Jr)returnc 8tjdtjdtjdtjdt di}|j t|}|s5t|tr|j}|Stjdd}|S)z4helper for read_ftps to map return codes to a number,iXiz9Unexpected exception type while connecting to ftp server.i) ftplib error_reply error_temp error_perm error_protoEOFErrorgettype isinstanceOSErrorerrnoLOGwarning)excftp_error_codescodes r'"ftp_get_return_code_from_exceptionrMKs C33C# O   tCy )D  c7 #99D K KKK D Kr)r%timeoutkwargs FtpResponsec Lt|}|jstdtd|t j 5}|j xsd}|jxsd}d|jk(rH tjt}tjd|||j|j||xsd tjd| |j#||j$xsdtjd|j' tjd|j,|j/d|j,|j0t3|j5|tjd|j7cdddS tj8} tjd||| j|j||xsd tjd|| j#||j$xsdtjd|j,| j/d|j,|j0t3|j5|tjd| j7cdddS#tj$r'}t!|} td |d | d | d||d}~wwxYw#tj($r*}tj+dtddd||d}~wwxYw#tj$r'}t!|} td|d | d | d||d}~wwxYw#tjd|j7wxYw#tj$r'}t!|} td|d | d | d||d}~wwxYw#tjd j7wxYw#1swYyxYw)azconnect to URL using ftp over TLS and read a file when using strict mode (ftps://), raise exception in event of failure when not using strict mode (ftp://), fall back to using unencrypted ftp url: string containing the desination to read a file from. The url is parsed with urllib.urlsplit to identify username, password, host, path, and port in the following format: ftps://[username:password@]host[:port]/[path] host is the only required component timeout: maximum time for the connection to take kwargs: unused, for compatibility with read_url returns: UrlResponse zInvalid url providedNcauserLheadersr% anonymousftps)contextz3Attempting to connect to %s via port [%s] over tls.@)hostportrNz1Reading file from server over tls failed for url z []z"Attempting to login with user [%s]r#)userpasswdzCreating a secure connectionzAttempted to connect to an insecure ftp server but used a scheme of ftps://, which is not allowed. Use ftp:// to allow connecting to insecure ftp servers.r<zReading file: %szRETR )callbackzClosing connectionz,Reading file from ftp server failed for url z(Attempting to connect to %s via port %s.)rhostnamerrioBytesIOr[usernamer r=FTP_TLSr rHdebugconnect all_errorsrMloginpasswordprot_pr@rIr2 retrbinarywriterPgetvaluecloseFTP) r%rNrO url_partsbufferr[r]ftp_tlserLftps r' read_ftpsrubs  I   (y$C   j~~#!!0[ Y%% %  ..24 I "++#Ns  II:D A  $--3 89 "  ,inn=""INN+,v||##6??#4c: ./ SjjV jjl >T "++#Ns  >E $--3 ,inn=y~~&67&,,O"6??#4c: ./ Ujj$$$ 9!<**-ba9  &$$  C N     .$$ 9!<++.%r$q:   ./ ($$ 9!<++.%r$q:   ./ Ujjs1P1AKPAL &A#M  %P9C'N3 %PL ""LL  P M %MM  P N"NNN  'N00P3O-"O((O--O00'PPP#r2 FileResponsec *|jdrtjd| tj|}t ||S#t $r}t|td||d}~wt$r}t||jd||d}~wwxYw)zmread a binary file and return a FileResponse matches function signature with read_ftps and read_url dataz'Unable to post data to file resource %sNrR) rCrHrIrload_binary_filervFileNotFoundErrorrrIOErrorrG)r2rOcontentsrss r' _read_filer}s zz& =tDM((.Hd++ OQY$GQN MQQWWdE1LMs# A BA'' B3B  B)rv UrlResponserPc ^|j} t|}|j}|dk(s |rd|dk(rt |j fi|S|dvr t|fi|S|dvr t|fi|Stjd|t|fi|S#t$r}t|||d}~wwxYw) a0Wrapper function around readurl to allow passing a file path as url. When url is not a local file path, passthrough any kwargs to readurl. In the case of parameter passthrough to readurl, default values for some parameters. See: call-signature of readurl in this module for param docs. )rSr%Nfiler,r)rtrW)rhttpszAttempting unknown protocol %s) lstripr ValueErrorrr r}r2rureadurlrHrI)r%rOparsedrsr s r'read_file_or_urlrs **,C0#]]F CC3q6M&++000 ? "''' $ $s%f%% 4f=s%f%% 0QC(a/0s B B,B''B,c eZdZddZdZdZy)StringResponsec<||_i|_||_||_yN)rLrTr|r%)selfr|r%rLs r'__init__zStringResponse.__init__s    r)c |jdk(S)NrLrargsrOs r'okzStringResponse.oksyyCr)c8|jjdS)Nzutf-8)r|decoders r'__str__zStringResponse.__str__s}}##G,,r)Nr)__name__ __module__ __qualname__rrrr4r)r'rrs  -r)rc*eZdZddedeffd ZxZS)rvr|r%c*t||||y)Nrsuperr)rr|r%rL __class__s r'rzFileResponse.__init__!s 3T2r)rrrrbytesr/r __classcell__rs@r'rvrv s33S33r)c(eZdZdedeffd ZxZS)rPr|r%c&t|||yrr)rr|r%rs r'rzFtpResponse.__init__&s 3'r)rrs@r'rPrP%s((S((r)ceZdZdejfdZedefdZede fdZ dde fdZ edZ edefdZd Z dd eed e deefd Zy )r~responsec||_yr) _response)rrs r'rzUrlResponse.__init__+s !r)r8c\|jjy|jjS)Nr))rcontentrs r'r|zUrlResponse.contents.s% >> ! ! )~~%%%r)c.|jjSr)rr%rs r'r%zUrlResponse.url4s~~!!!r)c@d}|rd}d|jcxkr|kryyy)Nr:r;rTFr)r redirects_okuppers r'rzUrlResponse.ok8s- E $)) #e # $r)c.|jjSr)rrTrs r'rTzUrlResponse.headersAs~~%%%r)c.|jjSr)r status_coders r'rLzUrlResponse.codeEs~~)))r)c.|jjSr)rtextrs r'rzUrlResponse.__str__Is~~"""r) chunk_sizedecode_unicodec#XK|jj||Ed{y7w)atIterates over the response data. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. :param chunk_size: Number of bytes it should read into memory. :param decode_unicode: If True, content will be decoded using the best available encoding based on the response. N)r iter_content)rrrs r'rzUrlResponse.iter_contentLs">>..z>JJJs *(*N)F)r!F)rrrrequestsResponserpropertyrr|r/r%boolrrTintrLrrr rr4r)r'r~r~*s"!2!2"&%&& "S""&&*c**#EJ K"3- K=A K % Kr)r~c >eZdZ ddedeedeedeefdZy)rNrSrLrTr%ctj|t|||_||_|in||_||_yr)r{rr/rSrLrTr%)rrSrLrTr%s r'rzUrlError.__init__\s: s5z*  &-o7 r))NNN) rrrr rrrr/rr4r)r'rr[sE#%)!  sm '"  c] r)ci}t|j}|dk(rE|rCd|vr|dr |d|d<nd|d<d|vrd|vr|d|dg|d<|Sd|vrt|d|d<|S)Nrca_certsverifyT cert_filekey_filecert)rr r/)r% ssl_detailsssl_argsr s r' _get_ssl_argsrjsH c] ! !F [  $Z)@!,Z!8HX !%HX  + %* *CK(J' HV  OK '";{#;|}|}Yd}~n4d}~wt4j@$r}t9||}|}d}Yd}~nd}~wwxYwtC|| }|s| xs|d z|k}|s|||xs|} |dkDs |rt$j'd|tEjF| 8)a`Wrapper around requests.Session to read the url and retry if necessary :param url: Mandatory url to request. :param data: Optional form data to post the URL. Will set request_method to 'POST' if present. :param timeout: Timeout in seconds to wait for a response. May be a tuple if specifying (connection timeout, read timeout). :param retries: Number of times to retry on exception if exception_cb is None or exception_cb returns True for the exception caught. Default is to fail with 0 retries on exception. :param sec_between: Default 1: amount of seconds passed to time.sleep between retries. None or -1 means don't sleep. :param headers: Optional dict of headers to send during request :param headers_cb: Optional callable returning a dict of values to send as headers during request :param headers_redact: Optional list of header names to redact from the log :param ssl_details: Optional dict providing key_file, ca_certs, and cert_file keys for use on in ssl connections. :param check_status: Optional boolean set True to raise when HTTPError occurs. Default: True. :param allow_redirects: Optional boolean passed straight to Session.request as 'allow_redirects'. Default: True. :param exception_cb: Optional callable to handle exception and returns True if retries are permitted. :param session: Optional exiting requests.Session instance to reuse. :param infinite: Bool, set True to retry indefinitely. Default: False. :param log_req_resp: Set False to turn off verbose debug messages. :param request_method: String passed as 'method' to Session.request. Typically GET, or POST. Default: POST if data is provided, GET otherwise. :param stream: if False, the response content will be immediately downloaded. )r%rrPOSTGETmethodNrNrr!z Cloud-Init/%srxz User-AgentrTz'[%s/%s] open '%s' with %s configurationrz(Read from %s (%s, %sb) after %s attempts)r%rLrTr%r1Please wait %s seconds while we wait to try againz"This path should be unreachable...r4)%r(updaterrEtuplemaxrrrversion_stringcopyrSessionr itemsrCdeepcopyrrHrerequestraise_for_statusrlenrr~rSSLErrorr HTTPErrorrrTRequestExceptionrrsleep RuntimeError)!r%rxrNrrrTrrrrrrrrrrrreq_args manual_tries user_agentifiltered_req_argskvmatched_headerskeyrrs url_errorraised_exceptionresponse_sleep_time will_retry sleep_times! r'rrsgj C.CH OOM#{34"1H  #'U'HX gu %")HY "%eGna"8HY L3wBHJ>+H99J>9J  J>#J99J>func.addreventdelayc>|r|j|ry|||S)z Execute func with optional delayrNNwaitrr rNr r s r'_run_func_with_delayrps' ::e: $  g r)333333? addresses stagger_delayc Nd}d}d}g}tj}tt|} t |D cic]$\} } | j t || ||| |z| &} } } t| |D]h} | | }| j}|r|}|j|/| j}|sB|j||fc| jdS|rtjd|||tjd|t!d cc} } w#t"$rFtjd d j%|d j%t't(|YnwxYw | jd||fS#| jdwxYw) aexecute multiple callbacks in parallel Run blocking func against two different addresses staggered with a delay. The first call to return successfully is returned from this function and remaining unfinished calls are cancelled if they have not yet started N) max_workersrr FrzD')D!*AD'<D'$AD''A E63F5E66FF$c>eZdZUeeed<eeed<eeed<y)HandledResponser%r wait_timeN)rrrrr/__annotations__r~rr4r)r'r-r-s# #{##r)r-inf) max_waitrN status_cbrrrr sleep_time_cbrconnect_synchronously async_delayr1r2rrr3rr4r5c T dtdtffd } ddtdtdtfdd ttd ttdt tt tffd d ttgt ttttffd ttttfdtdtdtdtf fd d tdtdtf fd dtffd } dtf fd }tj}r |r td| r| n|}|xs| }dd} |||||}d|j r"|j"|j j$fS|j&r'tj(|j&dzdo||}||r ydzt*j-d|tj(|tj}|r||z|zkDr||z z }|dkr y)a Wait for a response from one of the urls provided. :param urls: List of urls to try :param max_wait: Roughly the maximum time to wait before giving up The max time is *actually* len(urls)*timeout as each url will be tried once and given the timeout provided. a number <= 0 will always result in only one try :param timeout: Timeout provided to urlopen :param status_cb: Callable with string message when a url is not available :param headers_cb: Callable with single argument of url to get headers for request. :param headers_redact: List of header names to redact from the log :param sleep_time: Amount of time to sleep between retries. If this and sleep_time_cb are None, the default sleep time defaults to 1 second and increases by 1 seconds every 5 tries. Cannot be specified along with `sleep_time_cb`. :param exception_cb: Callable to handle exception and returns True if retries are permitted. :param sleep_time_cb: Callable with 2 arguments (response, loop_n) that generates the next sleep time. Cannot be specified along with 'sleep_time`. :param request_method: Indicates the type of HTTP request: GET, PUT, or POST :param connect_synchronously: If false, enables executing requests in parallel :param async_delay: Delay before parallel metadata requests, see RFC 6555 :return: tuple of (url, response contents), on failure, (False, None) :raises: UrlError on unrecoverable error loop_numberr8cS|dzdzS)Nr!r4)_r7rs r'default_sleep_timez(wait_for_url..default_sleep_time s'3zM9IA9MMr)rr1 start_timercj|tddfvry|dkxstj|z |z|kDS)z4Check if time is up based on start time and max waitr0NFr)rr monotonic)r1r<rs r'timeupzwait_for_url..timeups@ e d+ +A  NN z )J 6 A r)rr%cd}d}|r|sd}tt|}||fS |jj|js?d|jz}tt||j|j|}||fS|js;d|jz}tt||j|j|}||fS#tj j $rN}t||jj|jj|}|t|fcYd}~Sd}~wwxYw)z?Map requests response code/contents to internal "UrlError" typer#NzRequest timed outrzempty response [%s]zbad status code [%s])rrrrrrrrrrTr/r|rLr)rr%reasonurl_excrss r'handle_url_responsez)wait_for_url..handle_url_responses8S(Fz&12GF? " #    / / 1  *hmm8E>E url_reader_cburlsexc_cblog_cbcd}d} ||\}} ||\}}|st||dS tt j |z } rdznd} d|xs t |d d d | d | d |} || tddt|trt|| SdS#t$r} d| z}| }Yd} ~ d} ~ wt$r} d| z}| }Yd} ~ d} ~ wwxYw)z:Execute request, handle response, optionally log exceptionr#N)r.zrequest error [%s]zunexpected error [%s]z%ss unlimitedz Calling 'r%z ' failed [r,z]: rr%rr.) r-r Exceptionrrr>getattrrEr)rDrEr<rFrGrAr%rrBrs time_taken max_wait_str status_msgrCr1s r'read_url_handle_exceptionsz0wait_for_url..read_url_handle_exceptionsAs )$/MC1(C@OGV&sHEE)J67 +3ux'  177E51 1     zgx0gF;      ! )A-FG ,q0FG s#&B-- C6C CCCrNc:t|in||dS)NF)rTrrNrr)r)r%rNrrrs r' read_url_cbz!wait_for_url..read_url_cbms- $,B*S/))   r)cdtf fd }g}D]}tj} dk7r; s9 |rtdddcS r|z| zkDrt | z|z |||||}|j r|cS|j s|j|j |r t|nd} tdd| S)z|iterate over list of urls, request each one and handle responses and thrown exceptions individually per url r%c||fSrr4)r%rRrNs r'url_reader_serialz@wait_for_url..read_url_serial..url_reader_serial~sS'23 3r)rNrJ) r/rr>r-rrr.rr)r<rNrFrGrU wait_timesr%nowoutr.loop_nr1must_try_againrRrPr?rEs ` r'read_url_serialz%wait_for_url..read_url_serialws 43 4  1C.."C{>(J/* 44(w*x*?@":#8C"?@G,!3 FFC|| !!#--0+ 1,(2C Ot 4$)LLr)cBtt|}||||S)zpass list of urls to dual_stack which sends requests in parallel handle response and exceptions of the first endpoint to respond )rrN)rr+) r<rNrFrGurl_reader_parallelr5rRrPrEs r'read_url_parallelz'wait_for_url..read_url_parallels6 &  %   * z66  r)z3sleep_time and sleep_time_cb are mutually exclusiveNTFr!r)FN)r)rrrr~r/rrr r rrExceptionCallbackr-rr>rrr%r|r.rrHre)rEr1rNr2rrrrr3rr4r5r;r[r^r< do_read_urlcalculate_sleep_timerrespcurrent_sleep_time current_timerCrYrZrRrPr?s`` ``` ` ` @@@@@@r' wait_for_urlresS^N3N5N  E u '!;''!.6sm'! x!3& ''!R*  E5#(==> > * CcN# *  * " * *  * X  s { "M "M"MH   !JmNOO16G)>,>FH :w iH ==88T]]333 3 ^^ JJt~~ &aZF!N 1(FC (J(: ; " ! ?   %&~~' |g- X0EE,";>$2E2E#F""Xd.A.A-B#CD %d###G %yy} % % %  % % %  %s$B.0B B.B" B..B7c^|jsy|j}|i}|||<tjd|j5t |jd5}|j t j|ddddddy#1swYxYw#1swYyxYw)NzWriting wrx)rmrrrr|r}rlr~dumps)rrZvaluecurrs r'update_skew_filezOauthUrlHelper.update_skew_files"" !!# ;CD   $*=*=)>? @ &$   cC  &  HHTZZ_ % & & & & & &s$ B#!%BB#B B##B,ct|tr|jdk(s|jdk(syd|jvr!tj d|jy|jd} t jt|}t|t j z }t|jj}|jj|d}t!||z |j"kDr)|j%||tj d||||j|<y#t$r!}tj d||Yd}~yd}~wwxYw)Niidatez$Missing header 'date' in %s responsez#Failed to convert datetime '%s': %srz$Setting oauth clockskew for %s to %d)rErrLrTrHrIrrrrKrrr%netlocrsrCabsror)rrr remote_timersskewrZold_skews r'rzOauthUrlHelper.exception_cbs! y( +3&)..C*?  ** * KK>  O   ( ++io6K ;,- &-->>%%dA. x$ $"8"8 8  ! !$ - KK>d K#t  KK=tQ G  s/D<< E&E!!E&cL|jsiSd}t|j}|jr;||jvr-t t j |j|z}t ||j|j|j|j|S)N)r%rirkrlrj timestamp) rnrrrsrr oauth_headersrirkrlrj)rr%rrZs r'rzOauthUrlHelper.headers_cb*s~~I }## >>ddnn4DIIK(4>>$+??I**nn** 00   r)ct|j|jd|d<t|j|jd|d<||i|S)Nrr)r _headers_cbrC _exception_cb)r wrapped_funcrrOs r'_wrappedzOauthUrlHelper._wrapped<s\&   fjj6 |")    > :" ~T,V,,r)c0|jt||Sr)rrers r'rezOauthUrlHelper.wait_for_urlEs}}\488r)c0|jt||Sr)rrrs r'rzOauthUrlHelper.readurlHs}}WdF33r)cnd} |r||}|j||S#|j|wxYw)NTr)rextra_exception_cbrrets r'rzOauthUrlHelper._exception_cbKs> )!(3   i (    i (s !4c^i}|r||}|j|j||Sr)rr)rextra_headers_cbr%rTs r'rzOauthUrlHelper._headers_cbTs. &s+Gts+,r))NNNNz/run/oauth_skew.json) rrrrrrrrrrrerrrr4r)r'rgrgsE- #6 &8 $-94r)rgc ddlm}|r t |}nd}|j |||||j |}|j|\} } } | S#t$r}td|d}~wwxYw)Nrzoauth support is not available) client_secretresource_owner_keyresource_owner_secretsignature_methodr)oauthlib.oauth1oauth1 ImportErrorNotImplementedErrorr/ClientSIGNATURE_PLAINTEXTsign) r%rirkrlrjrrrsclient_urisigned_headers_bodys r'rr\sK( N  ]]%$*33 F#)++c"2D.% # K!"BCJKsA A* A%%A*)rYr)r )Mrr=rar~loggingrzrrconcurrent.futuresrrr email.utilsr functoolsr http.clientr itertoolsr sslr typingr r r rrrrrr urllib.parserrrrrr cloudinitrrr getLoggerrrHrrr_r(r7rrMr/rdictrur}rrrvrPr~r{rrrrrrrr+r-rerergrr4r)r'rs^   MM!!&   ?>00g! XzlD&89:"  s.3D MS M~ M & 78&< - -3>3 (.( .K.Kb w $#%J'+ $e_ J    &*  %m=$m=$ %m=&'m=j"  38   ??  E?   *! P- 38 P-CyP-P- P-  8C=(;/ /0 P-fjEl#))%)"&&*=A"&||e_ |  | " ||$|Hc5\5%89:|| ||~||@LPr)