13#include <zypp-core/zyppng/base/EventDispatcher> 
   15#include <zypp-core/zyppng/core/String> 
   18#include <zypp-curl/CurlConfig> 
   19#include <zypp-curl/auth/CurlAuthData> 
   20#include <zypp-media/MediaConfig> 
   31#include <boost/variant.hpp> 
   32#include <boost/variant/polymorphic_get.hpp> 
   38    static size_t nwr_headerCallback (  
char *ptr, 
size_t size, 
size_t nmemb, 
void *userdata  ) {
 
   45    static size_t nwr_writeCallback ( 
char *ptr, 
size_t size, 
size_t nmemb, 
void *userdata ) {
 
   54    template<
class T> 
struct always_false : std::false_type {};
 
   78    , 
_headers( 
std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( nullptr, &curl_slist_free_all ) )
 
 
  108    const std::string urlScheme = 
_url.getScheme();
 
  109    if ( urlScheme == 
"http" ||  urlScheme == 
"https" )
 
  121      std::string urlBuffer( 
_url.asString() );
 
  150        const auto &cHeaders = 
_dispatcher->hostSpecificHeaders();
 
  151        if ( 
auto i = cHeaders.find(
_url.getHost()); i != cHeaders.end() ) {
 
  152          for ( 
const auto &[key, value] : i->second ) {
 
  154              "%s: %s", key.c_str(), value.c_str() )
 
  165        case 4: 
setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 ); 
break;
 
  166        case 6: 
setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6 ); 
break;
 
  170      setCurlOption( CURLOPT_HEADERFUNCTION, &nwr_headerCallback );
 
  185      if ( urlScheme == 
"https" )
 
  206#ifdef CURLSSLOPT_ALLOW_BEAST 
  208        setCurlOption( CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
 
  234        std::string use_auth = 
_settings.authType();
 
  235        if (use_auth.empty())
 
  236          use_auth = 
"digest,basic";    
 
  238        if( auth != CURLAUTH_NONE)
 
  240          DBG << 
_easyHandle << 
" "  << 
"Enabling HTTP authentication methods: " << use_auth
 
  241              << 
" (CURLOPT_HTTPAUTH=" << auth << 
")" << std::endl;
 
  250        setCurlOption(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
 
  261        if ( proxyuserpwd.empty() )
 
  266            DBG << 
_easyHandle << 
" "  << 
"Proxy: ~/.curlrc does not contain the proxy-user option" << std::endl;
 
  270            DBG << 
_easyHandle << 
" " << 
"Proxy: using proxy-user from ~/.curlrc" << std::endl;
 
  278        if ( ! proxyuserpwd.empty() )
 
  283#if CURLVERSION_AT_LEAST(7,19,4) 
  288        DBG << 
_easyHandle << 
" "  << 
"Proxy: explicitly NOPROXY" << std::endl;
 
  303#if CURLVERSION_AT_LEAST(7,15,5) 
  315#if CURLVERSION_AT_LEAST(7,18,0) 
  322      for ( 
const auto &header : locSet.
headers() ) {
 
  323        if ( !z_func()->addRequestHeader( header.c_str() ) )
 
  335            return ( elem1._start < elem2._start );
 
  339          if ( 
auto initState = std::get_if<pending_t>(&
_runningMode) ) {
 
  342            initState->_partialHelper = std::make_unique<CurlMultiPartHandler>(
 
  348            helper = initState->_partialHelper.get();
 
  350          } 
else if ( 
auto pendingState = std::get_if<prepareNextRangeBatch_t>(&
_runningMode) ) {
 
  351            helper = pendingState->_partialHelper.get();
 
  353            errBuf = 
"Request is in invalid state to call setupHandle";
 
 
  375    auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &
_runningMode );
 
  377      DBG << 
_easyHandle << 
"Can only create output file in running mode" << std::endl;
 
  381    if ( !rmode->_outFile ) {
 
  382      std::string openMode = 
"w+b";
 
  386      rmode->_outFile = fopen( 
_targetFile.asString().c_str() , openMode.c_str() );
 
  390        rmode->_outFile = fopen( 
_targetFile.asString().c_str() , 
"w+b" );
 
  393      if ( !rmode->_outFile ) {
 
 
  406    auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &
_runningMode );
 
  407    if ( rmode->_partialHelper ) 
return rmode->_partialHelper->canRecover();
 
 
  413    auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &
_runningMode );
 
  415      errBuf = 
"NetworkRequestPrivate::prepareToContinue called in invalid state";
 
  419    if ( rmode->_partialHelper && rmode->_partialHelper->hasMoreWork() ) {
 
  425      auto &prepMode = std::get<prepareNextRangeBatch_t>(
_runningMode);
 
  426      if ( !prepMode._partialHelper->prepareToContinue() ) {
 
  427        errBuf = prepMode._partialHelper->lastErrorMessage();
 
  431      if ( hadRangeFail ) {
 
  445    errBuf = 
"Request has no more work";
 
 
  453    return std::any_of( 
_requestedRanges.begin(), 
_requestedRanges.end(), []( 
const auto &range ){ return range._rangeState == CurlMultiPartHandler::Pending; });
 
 
  458    bool isRangeContinuation = std::holds_alternative<prepareNextRangeBatch_t>( 
_runningMode );
 
  459    if ( isRangeContinuation ) {
 
  460      MIL << 
_easyHandle << 
" " << 
"Continuing a previously started range batch." << std::endl;
 
  468    if ( m._activityTimer ) {
 
  471      m._activityTimer->start( 
static_cast<uint64_t
>( 
_settings.timeout() * 1000 ) );
 
  474    if ( !isRangeContinuation )
 
 
  480    if ( std::holds_alternative<running_t>(
_runningMode) ) {
 
  482      if ( rmode._partialHelper )
 
  483        rmode._partialHelper->finalize();
 
 
  490    resState.
_result = std::move(err);
 
  492    if ( std::holds_alternative<running_t>(
_runningMode) ) {
 
  501          if ( !rmode._partialHelper->verifyData() ){
 
  508            if ( fseek( rmode._outFile, 0, SEEK_SET ) != 0 ) {
 
  511              constexpr size_t bufSize = 4096;
 
  513              while( 
auto cnt = fread(buf, 1, bufSize, rmode._outFile ) > 0 ) {
 
  525        if ( calcSum != expSum  ) {
 
  534      rmode._outFile.reset();
 
 
  559    std::map<std::string, boost::any> extraInfo;
 
  560    extraInfo.insert( {
"requestUrl", 
_url } );
 
 
  572    if ( std::holds_alternative<running_t>( 
_runningMode ) ){
 
  574      if ( rmode._activityTimer && rmode._activityTimer->isRunning() )
 
  575        rmode._activityTimer->start();
 
 
  585    if ( !std::holds_alternative<running_t>(that->
_runningMode) ){
 
  586      DBG << that->
_easyHandle << 
" " << 
"Curl progress callback was called in invalid state "<< that->z_func()->state() << std::endl;
 
  590    auto &rmode = std::get<running_t>( that->
_runningMode );
 
  595    rmode._isInCallback = 
true;
 
  596    if ( rmode._lastProgressNow != dlnow ) {
 
  597      rmode._lastProgressNow = dlnow;
 
  598      that->
_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow );
 
  600    rmode._isInCallback = 
false;
 
  602    return rmode._cachedResult ? CURLE_ABORTED_BY_CALLBACK : CURLE_OK;
 
 
  615      std::string_view hdr( ptr, bytes );
 
  617      hdr.remove_prefix( std::min( hdr.find_first_not_of(
" \t\r\n"), hdr.size() ) );
 
  618      const auto lastNonWhitespace = hdr.find_last_not_of(
" \t\r\n");
 
  619      if ( lastNonWhitespace != hdr.npos )
 
  620        hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
 
  622        hdr = std::string_view();
 
  629        const auto &repSize = rmode._partialHelper->reportedFileSize ();
 
  635      if ( zypp::strv::hasPrefixCI( hdr, 
"HTTP/" ) ) {
 
  638        (void)curl_easy_getinfo( 
_easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
 
  644      } 
else if ( zypp::strv::hasPrefixCI( hdr, 
"Location:" ) ) {
 
  648      } 
else if ( zypp::strv::hasPrefixCI( hdr, 
"Content-Length:") )  {
 
  650        auto str = std::string ( lenStr.data(), lenStr.length() );
 
  653          DBG << 
_easyHandle << 
" " << 
"Got Content-Length Header: " << len << std::endl;
 
 
  683      if ( fseek( rmode._outFile, *offset, SEEK_SET ) != 0 ) {
 
  685          "Unable to set output file pointer." );
 
  688      rmode._currentFileOffset = *offset;
 
  692      const auto &repSize = rmode._partialHelper->reportedFileSize ();
 
  705    auto written = fwrite( data, 1, max, rmode._outFile );
 
  714    rmode._currentFileOffset += written;
 
  717    rmode._downloaded += written;
 
 
  725    auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &
_runningMode );
 
  726    if ( !rmode || !rmode->_partialHelper || !rmode->_partialHelper->hasError() )
 
  730    if ( rmode->_cachedResult )
 
  734    MIL_MEDIA << 
_easyHandle << 
" Multipart handler announced error code " << lastErr.toString () <<  std::endl;
 
  735    rmode->_cachedResult = lastErr;
 
 
  749    if ( d->_dispatcher )
 
  750      d->_dispatcher->cancel( *
this, 
"Request destroyed while still running" );
 
 
  760    return d_func()->_expectedFileSize;
 
 
  767    if ( 
state() == 
Pending && triggerReschedule && d->_dispatcher )
 
  768      d->_dispatcher->reschedule();
 
 
  773    return d_func()->_priority;
 
 
  778    d_func()->_options = opt;
 
 
  783    return d_func()->_options;
 
 
  792    d->_requestedRanges.push_back( 
Range::make( start, len, std::move(digest), std::move( expectedChkSum ), std::move( userData ), digestCompareLen, chksumpad ) );
 
 
  801    d->_requestedRanges.push_back( std::move(range) );
 
  802    auto &rng = d->_requestedRanges.back();
 
  804    rng.bytesWritten = 0;
 
  806      rng._digest->reset();
 
 
  820        ._fileDigest   = std::move(fDig),
 
 
  831    d->_requestedRanges.clear();
 
 
  836    const auto mystate = 
state();
 
  842    std::vector<Range> failed;
 
  843    for ( 
auto &r : d->_requestedRanges ) {
 
  845        failed.push_back( r.clone() );
 
 
  853    return d_func()->_requestedRanges;
 
 
  858    return d_func()->_lastRedirect;
 
 
  863    return d_func()->_easyHandle;
 
 
  868    const auto myerr = 
error();
 
  869    const auto mystate = 
state();
 
  875    auto getMeasurement = [ this ]( 
const CURLINFO info, std::chrono::microseconds &
target ){
 
  876      using FPSeconds = std::chrono::duration<double, std::chrono::seconds::period>;
 
  878      const auto res = curl_easy_getinfo( d_func()->_easyHandle, info, &val );
 
  879      if ( CURLE_OK == res ) {
 
  880        target = std::chrono::duration_cast<std::chrono::microseconds>( FPSeconds(val) );
 
  884    getMeasurement( CURLINFO_NAMELOOKUP_TIME, t.
namelookup );
 
  885    getMeasurement( CURLINFO_CONNECT_TIME, t.
connect);
 
  886    getMeasurement( CURLINFO_APPCONNECT_TIME, t.
appconnect);
 
  887    getMeasurement( CURLINFO_PRETRANSFER_TIME , t.
pretransfer);
 
  888    getMeasurement( CURLINFO_TOTAL_TIME, t.
total);
 
  889    getMeasurement( CURLINFO_REDIRECT_TIME, t.
redirect);
 
 
  898    if ( !std::holds_alternative<NetworkRequestPrivate::running_t>( d->_runningMode) )
 
  901    const auto &rmode = std::get<NetworkRequestPrivate::running_t>( d->_runningMode );
 
 
  907    return d_func()->_url;
 
 
  921    return d_func()->_targetFile;
 
 
  929    d->_targetFile = path;
 
 
  934    return d_func()->_fMode;
 
 
  942    d->_fMode = std::move( mode );
 
 
  948    if ( curl_easy_getinfo( d_func()->_easyHandle, CURLINFO_CONTENT_TYPE, &ptr ) == CURLE_OK && ptr )
 
  949      return std::string(ptr);
 
  950    return std::string();
 
 
  957      if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
 
  959      else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
 
  960                         || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
 
  961        return arg._contentLenght;
 
  963        static_assert(always_false<T>::value, 
"Unhandled state type");
 
  964    }, d_func()->_runningMode);
 
 
  971      if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
 
  973      else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
 
  974                          || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t>
 
  975                          || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
 
  976        return arg._downloaded;
 
  978        static_assert(always_false<T>::value, 
"Unhandled state type");
 
  979    }, d_func()->_runningMode);
 
 
  984    return d_func()->_settings;
 
 
  989    return std::visit([
this](
auto& arg) {
 
  991      if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
 
  993      else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
 
  995      else if constexpr (std::is_same_v<T, NetworkRequestPrivate::finished_t>) {
 
  996        if ( std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode )._result.isError() )
 
 1002        static_assert(always_false<T>::value, 
"Unhandled state type");
 
 1003    }, d_func()->_runningMode);
 
 
 1008    const auto s = 
state();
 
 1011    return std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode)._result;
 
 
 1017      return std::string();
 
 
 1031    curl_slist *res = curl_slist_append( d->_headers ? d->_headers.get() : 
nullptr, header.c_str() );
 
 1036      d->_headers = std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( res, &curl_slist_free_all );
 
 
 1043    return d_func()->_currentCookieFile;
 
 
 1048    d_func()->_currentCookieFile = std::move(
cookieFile);
 
 
 1053    return d_func()->_sigStarted;
 
 
 1058    return d_func()->_sigBytesDownloaded;
 
 
 1063    return d_func()->_sigProgress;
 
 
 1068    return d_func()->_sigFinished;
 
 
Store and operate with byte count.
static const Unit B
1 Byte
Compute Message Digests (MD5, SHA1 etc)
bool create(const std::string &name)
initialize creation of a new message digest
Base class for Exception.
std::string asString() const
Error message provided by dumpOn as string.
const char * c_str() const
String representation.
bool empty() const
Test for an empty path.
The CurlMultiPartHandler class.
const std::string & lastErrorMessage() const
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})
The NetworkRequestError class Represents a error that occured in.
Type type() const
type Returns the type of the error
std::string nativeErrorString() const
bool isError() const
isError Will return true if this is a actual error
size_t headerfunction(char *ptr, size_t bytes) override
std::optional< FileVerifyInfo > _fileVerification
The digest for the full file.
enum zyppng::NetworkRequestPrivate::ProtocolMode _protocolMode
void notifyErrorCodeChanged() override
zypp::Pathname _targetFile
void resetActivityTimer()
zypp::Pathname _currentCookieFile
Signal< void(NetworkRequest &req, zypp::ByteCount count)> _sigBytesDownloaded
NetworkRequestDispatcher * _dispatcher
std::vector< NetworkRequest::Range > _requestedRanges
the requested ranges that need to be downloaded
size_t writefunction(char *ptr, std::optional< off_t > offset, size_t bytes) override
static int curlProgressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
std::string errorMessage() const
Signal< void(NetworkRequest &req)> _sigStarted
NetworkRequest::FileMode _fMode
std::variant< pending_t, running_t, prepareNextRangeBatch_t, finished_t > _runningMode
bool initialize(std::string &errBuf)
void onActivityTimeout(Timer &)
Signal< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> _sigProgress
std::string _lastRedirect
to log/report redirections
NetworkRequest::Options _options
bool prepareToContinue(std::string &errBuf)
void setResult(NetworkRequestError &&err)
~NetworkRequestPrivate() override
std::array< char, CURL_ERROR_SIZE+1 > _errorBuf
bool setupHandle(std::string &errBuf)
NetworkRequestPrivate(Url &&url, zypp::Pathname &&targetFile, NetworkRequest::FileMode fMode, NetworkRequest &p)
TransferSettings _settings
void setCurlOption(CURLoption opt, T data)
zypp::ByteCount _expectedFileSize
Signal< void(NetworkRequest &req, const NetworkRequestError &err)> _sigFinished
std::unique_ptr< curl_slist, decltype(&curl_slist_free_all) > _headers
bool setExpectedFileChecksum(const zypp::CheckSum &expected)
zypp::ByteCount reportedByteCount() const
Returns the number of bytes that are reported from the backend as the full download size,...
const zypp::Pathname & targetFilePath() const
Returns the target filename path.
zypp::ByteCount downloadedByteCount() const
Returns the number of already downloaded bytes as reported by the backend.
void resetRequestRanges()
void setUrl(const Url &url)
This will change the URL of the request.
void setExpectedFileSize(zypp::ByteCount expectedFileSize)
void setPriority(Priority prio, bool triggerReschedule=true)
std::vector< char > peekData(off_t offset, size_t count) const
std::string contentType() const
Returns the content type as reported from the server.
void setFileOpenMode(FileMode mode)
Sets the file open mode to mode.
bool addRequestHeader(const std::string &header)
~NetworkRequest() override
void setOptions(Options opt)
FileMode fileOpenMode() const
Returns the currently configured file open mode.
zypp::ByteCount expectedFileSize() const
bool hasError() const
Checks if there was a error with the request.
State state() const
Returns the current state the HttpDownloadRequest is in.
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
std::optional< Timings > timings() const
After the request is finished query the timings that were collected during download.
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
NetworkRequest(Url url, zypp::Pathname targetFile, FileMode fMode=WriteExclusive)
Priority priority() const
NetworkRequestError error() const
Returns the last set Error.
void setTargetFilePath(const zypp::Pathname &path)
Changes the target file path of the download.
const zypp::Pathname & cookieFile() const
void * nativeHandle() const
void setCookieFile(zypp::Pathname cookieFile)
void addRequestRange(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes expectedChkSum=CheckSumBytes(), std::any userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > chksumpad={})
std::vector< Range > failedRanges() const
const std::vector< Range > & requestedRanges() const
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
SignalProxy< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> sigProgress()
Signals if there was data read from the download.
TransferSettings & transferSettings()
CurlMultiPartHandler::Range Range
const std::string & lastRedirectInfo() const
The Timer class provides repetitive and single-shot timers.
SignalProxy< void(Timer &t)> sigExpired()
This signal is always emitted when the timer expires.
uint64_t remaining() const
uint64_t interval() const
#define EXPLICITLY_NO_PROXY
std::string curlUnEscape(const std::string &text_r)
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
CURLcode setCurlRedirProtocols(CURL *curl)
typename decay< T >::type decay_t
String related utilities and Regular expression matching.
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Types and functions for filesystem operations.
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
std::vector< char > peek_data_fd(FILE *fd, off_t offset, size_t count)
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
TInt strtonum(const C_Str &str)
Parsing numbers from string.
std::string trim(const std::string &s, const Trim trim_r)
Easy-to use interface to the ZYPP dependency resolver.
T trim(StrType &&s, const Trim trim_r)
zypp::media::TransferSettings TransferSettings
std::string strerr_cxx(const int err=-1)
static Range make(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
NetworkRequestError _result
zypp::ByteCount _contentLenght
std::unique_ptr< CurlMultiPartHandler > _partialHelper
running_t(pending_t &&prevState)
std::chrono::microseconds appconnect
std::chrono::microseconds redirect
std::chrono::microseconds pretransfer
std::chrono::microseconds total
std::chrono::microseconds namelookup
std::chrono::microseconds connect
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
#define ZYPP_IMPL_PRIVATE(Class)