33#include <zypp-curl/parser/MetaLinkParser> 
   36#include <zypp-curl/auth/CurlAuthData> 
   43#undef CURLVERSION_AT_LEAST 
   44#define CURLVERSION_AT_LEAST(M,N,O) LIBCURL_VERSION_NUM >= ((((M)<<8)+(N))<<8)+(O) 
  187  void adddnsfd( std::vector<GPollFD> &waitFds );
 
  188  void dnsevent(
const std::vector<GPollFD> &waitFds );
 
  219  size_t writefunction  ( 
char *ptr, std::optional<off_t> offset, 
size_t bytes ) 
override;
 
  221  bool   beginRange     ( off_t range, std::string &cancelReason ) 
override;
 
  222  bool   finishedRange  ( off_t range, 
bool validated, std::string &cancelReason ) 
override;
 
 
  235                    Url baseurl, CURLM *multi, FILE *fp,
 
  244  void run(std::vector<Url> &urllist);
 
  266  std::list< std::unique_ptr<multifetchworker> > 
_workers;
 
 
  319#if _POSIX_C_SOURCE >= 199309L 
  321  if ( clock_gettime( CLOCK_MONOTONIC, &ts) )
 
  323  return ts.tv_sec + ts.tv_nsec / 1000000000.;
 
  326  if (gettimeofday(&tv, NULL))
 
  328  return tv.tv_sec + tv.tv_usec / 1000000.;
 
 
  336    return bytes ? 0 : 1;
 
  358  const auto &blk = 
_blocks[*currRange];
 
  359  off_t seekTo = blk._start + blk.bytesWritten;
 
  361  if ( ftell( 
_request->_fp ) != seekTo ) {
 
  363    if (fseeko(
_request->_fp, seekTo, SEEK_SET))
 
  364      return bytes ? 0 : 1;
 
  367  size_t cnt = fwrite(ptr, 1, bytes, 
_request->_fp);
 
 
  376  const auto &currRangeState = stripeDesc.blockStates[stripeRangeOff];
 
  379    cancelReason = 
"Cancelled because stripe block is already finalized";
 
  381    WAR << 
"#" << 
_workerno << 
": trying to start a range ("<<stripeRangeOff<<
"["<< 
_blocks[workerRangeOff]._start <<
" : "<<
_blocks[workerRangeOff]._len<<
"]) that was already finalized, cancelling. Stealing was: " << 
_request->_stealing << endl;
 
 
  392  const auto &currRangeState = stripeDesc.blockStates[stripeRangeOff];
 
  396    cancelReason = 
"Block failed to validate";
 
  412      WAR << 
"#" << 
_workerno << 
": Broken data in COMPETING block, requesting refetch. Stealing is:  " << 
_request->_stealing << endl;
 
 
  423  if (l > 9 && !strncasecmp(p, 
"Location:", 9)) {
 
  424    std::string line(p + 9, l - 9);
 
  425    if (line[l - 10] == 
'\r')
 
  426      line.erase(l - 10, 1);
 
  427    XXX << 
"#" << 
_workerno << 
": redirecting to" << line << endl;
 
  432  if ( repSize && *repSize != 
_request->_filesize  ) {
 
  433      XXX << 
"#" << 
_workerno << 
": filesize mismatch" << endl;
 
 
  452    XXX << 
"reused worker from pool" << endl;
 
  460  if ( 
url.getScheme() == 
"http" ||  
url.getScheme() == 
"https" )
 
 
  472    curl_easy_cleanup(
_curl);
 
  478  curl_easy_setopt(
_curl, CURLOPT_PRIVATE, 
this);
 
  484  if ( 
_url.getHost() == 
_request->_context->_url.getHost()) {
 
  489      curl_easy_setopt(
_curl, CURLOPT_USERPWD, 
_settings.userPassword().c_str());
 
  490      std::string use_auth = 
_settings.authType();
 
  491      if (use_auth.empty())
 
  492        use_auth = 
"digest,basic";        
 
  494      if( auth != CURLAUTH_NONE)
 
  496        XXX << 
"#" << 
_workerno << 
": Enabling HTTP authentication methods: " << use_auth
 
  497            << 
" (CURLOPT_HTTPAUTH=" << auth << 
")" << std::endl;
 
  498        curl_easy_setopt(
_curl, CURLOPT_HTTPAUTH, auth);
 
 
  513#if CURLVERSION_AT_LEAST(7,15,5) 
  514          curl_easy_setopt(
_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)0);
 
  516          curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
 
  517          curl_easy_setopt(
_curl, CURLOPT_WRITEFUNCTION, (
void *)0);
 
  518          curl_easy_setopt(
_curl, CURLOPT_WRITEDATA, (
void *)0);
 
  519          curl_easy_setopt(
_curl, CURLOPT_HEADERFUNCTION, (
void *)0);
 
  520          curl_easy_setopt(
_curl, CURLOPT_HEADERDATA, (
void *)0);
 
  524        curl_easy_cleanup(
_curl);
 
 
  546  const char *s = getenv(name.c_str());
 
  547  return s && *s ? true : 
false;
 
 
  553  std::string host = 
_url.getHost();
 
  558  if (
_request->_context->isDNSok(host))
 
  563  if (inet_pton(AF_INET, host.c_str(), addrbuf) == 1)
 
  565  if (inet_pton(AF_INET6, host.c_str(), addrbuf) == 1)
 
  573  std::string schemeproxy = 
_url.getScheme() + 
"_proxy";
 
  576  if (schemeproxy != 
"http_proxy")
 
  578      std::transform(schemeproxy.begin(), schemeproxy.end(), schemeproxy.begin(), ::toupper);
 
  583  XXX << 
"checking DNS lookup of " << host << endl;
 
  592  if (
_pid == pid_t(-1))
 
  605      struct addrinfo *ai = 
nullptr, aihints;
 
  606      memset(&aihints, 0, 
sizeof(aihints));
 
  607      aihints.ai_family = PF_UNSPEC;
 
  608      int tstsock = socket(PF_INET6, SOCK_DGRAM | SOCK_CLOEXEC, 0);
 
  610        aihints.ai_family = PF_INET;
 
  613      aihints.ai_socktype = SOCK_STREAM;
 
  614      aihints.ai_flags = AI_CANONNAME;
 
  615      unsigned int connecttimeout = 
_request->_connect_timeout;
 
  617        alarm(connecttimeout);
 
  618      signal(SIGALRM, SIG_DFL);
 
  619      if (getaddrinfo(host.c_str(), NULL, &aihints, &ai))
 
 
  637          .events = G_IO_IN | G_IO_HUP | G_IO_ERR,
 
 
  645  bool hasEvent = std::any_of( waitFds.begin (), waitFds.end(),[
this]( 
const GPollFD &waitfd ){
 
  646    return ( waitfd.fd == _dnspipe && waitfd.revents != 0 );
 
  659  if (!WIFEXITED(status))
 
  666  int exitcode = WEXITSTATUS(status);
 
  667  XXX << 
"#" << 
_workerno << 
": DNS lookup returned " << exitcode << endl;
 
 
  685  auto &blk = 
_blocks[workerRangeIdx];
 
  689  const auto currOf = ftell( 
_request->_fp );
 
  693  if (fseeko(
_request->_fp, blk._start, SEEK_SET))
 
  701    size_t cnt = l > 
sizeof(buf) ? 
sizeof(buf) : l;
 
  702    if (fread(buf, cnt, 1, 
_request->_fp) != 1)
 
  708  if (fseeko(
_request->_fp, currOf, SEEK_SET))
 
  711  blk._digest = std::move(newDig);
 
 
  726  std::optional<zypp::Digest> digest;
 
  727  std::optional<size_t> relDigLen;
 
  728  std::optional<size_t> blkSumPad;
 
  730  const auto &blk   = 
_request->_blklist.getBlock( blkNo );
 
  731  if ( 
_request->_blklist.haveChecksum ( blkNo ) ) {
 
  732    sum = 
_request->_blklist.getChecksum( blkNo );
 
  733    relDigLen = sum.size( );
 
  734    blkSumPad = 
_request->_blklist.checksumPad( );
 
  736    digest->create( 
_request->_blklist.getChecksumType() );
 
  739  return MultiByteHandler::Range::make(
 
  745        std::move(relDigLen),
 
  746        std::move(blkSumPad) );
 
 
  753      XXX << 
"start stealing!" << endl;
 
  762  for (
auto workeriter = 
_request->_workers.begin(); workeriter != 
_request->_workers.end(); ++workeriter)
 
  767      if (worker->
_pass == -1)
 
  834          XXX << 
"#" << 
_workerno << 
": going to sleep for " << sl * 1000 << 
" ms" << endl;
 
 
  855  for ( 
auto workeriter = 
_request->_workers.begin(); workeriter != 
_request->_workers.end(); ++workeriter)
 
 
  891  for ( uint i = 0; i < stripeDesc.blocks.size(); i++ ) {
 
  911  DBG << 
"#" << 
_workerno << 
"Done adding blocks to download, going to download: " << 
_blocks.size() << 
" nr of block with " << 
_datasize << 
" nr of bytes" << std::endl;
 
 
  922  bool hadRangeFail = 
_multiByteHandler->lastError() == MultiByteHandler::Code::RangeFail;
 
  928  if ( hadRangeFail ) {
 
  935    curl_easy_reset( 
_curl );
 
 
  957  if (curl_multi_add_handle(
_request->_multi, 
_curl) != CURLM_OK) {
 
 
  973  : 
internal::CurlPollHelper::CurlPoll{ multi }
 
  982  , 
_timeout(context->_settings.timeout())
 
  984  , 
_maxspeed(context->_settings.maxDownloadSpeed())
 
  985  , 
_maxworkers(context->_settings.maxConcurrentConnections())
 
  994  for (
size_t blkno = 0; blkno < 
_blklist.numBlocks(); blkno++)
 
 1002  for (
size_t blkno = 0; blkno < 
_blklist.numBlocks(); blkno++) {
 
 1012    currStripeSize += blk.
size;
 
 
 1027  std::vector<Url>::iterator urliter = urllist.begin();
 
 1033  if (mcode != CURLM_OK)
 
 1039      std::vector<GPollFD> waitFds;
 
 1044          XXX << 
"finished!" << endl;
 
 1051          _workers.push_back(std::make_unique<multifetchworker>(workerno++, *
this, *urliter));
 
 1068          WAR << 
"No more active workers!" << endl;
 
 1070          for (
auto workeriter = 
_workers.begin(); workeriter != 
_workers.end(); ++workeriter)
 
 1080        for (
auto workeriter = 
_workers.begin(); workeriter != 
_workers.end(); ++workeriter)
 
 1081          (*workeriter)->adddnsfd( waitFds );
 
 1090          for (
auto workeriter = 
_workers.begin(); workeriter != 
_workers.end(); ++workeriter) {
 
 1104          timeoutMs = sl * 1000;
 
 1108        timeoutMs = std::min<long>( timeoutMs, _curlHelper.
timeout_ms.value() );
 
 1110      dnsFdCount = waitFds.size(); 
 
 1111      waitFds.insert( waitFds.end(), _curlHelper.
socks.begin(), _curlHelper.
socks.end() ); 
 
 1114      if ( !waitFds.empty() || timeoutMs != -1) {
 
 1119          for (
auto workeriter = 
_workers.begin(); workeriter != 
_workers.end(); ++workeriter)
 
 1124              (*workeriter)->dnsevent( waitFds );
 
 1133          if (mcode != CURLM_OK)
 
 1137          if (mcode != CURLM_OK)
 
 1162          for (
auto workeriter = 
_workers.begin(); workeriter != 
_workers.end(); ++workeriter)
 
 1171              XXX << 
"#" << worker->
_workerno << 
": sleep done, wake up" << endl;
 
 1179      CURLMsg *msg = 
nullptr;
 
 1181      while ((msg = curl_multi_info_read(
_multi, &nqueue)) != 0)
 
 1183          if (msg->msg != CURLMSG_DONE)
 
 1185          CURL *easy = msg->easy_handle;
 
 1186          CURLcode cc = msg->data.result;
 
 1189          if (curl_easy_getinfo(easy, CURLINFO_PRIVATE, &worker) != CURLE_OK)
 
 1200          curl_multi_remove_handle(
_multi, easy);
 
 1202          const auto &setWorkerBroken = [&]( 
const std::string &
str = {} ){
 
 1204            if ( !
str.empty () )
 
 1215            WAR << 
"#" << worker->
_workerno << 
": has no multibyte handler, this is a bug" << endl;
 
 1216            setWorkerBroken(
"Multibyte handler error");
 
 1225            WAR << 
"#" << worker->
_workerno << 
": still has work to do or can recover from a error, continuing the job!" << endl;
 
 1236          if ( cc != CURLE_OK ) {
 
 1242              WAR << 
"#" << worker->
_workerno << 
": failed, but was set to discard, reusing for new requests" << endl;
 
 1261              bool done = std::all_of( wrkerStripe.blockStates.begin(), wrkerStripe.blockStates.begin(), []( 
const Stripe::RState s ) { return s == Stripe::FINALIZED; } );
 
 1264                std::for_each( wrkerStripe.blockStates.begin(), wrkerStripe.blockStates.begin(), []( 
Stripe::RState &s ) {
 
 1265                  if ( s != Stripe::FINALIZED)
 
 1266                    s = Stripe::PENDING;
 
 1277            int maxworkerno = 0;
 
 1279            for (
auto workeriter = 
_workers.begin(); workeriter != 
_workers.end(); ++workeriter)
 
 1294                double ratio = worker->
_avgspeed / maxavg;
 
 1297                  ratio = ratio * ratio;
 
 1300                    XXX << 
"#" << worker->
_workerno << 
": too slow ("<< ratio << 
", " << worker->
_avgspeed << 
", #" << maxworkerno << 
": " << maxavg << 
"), going to sleep for " << ratio * 1000 << 
" ms" << endl;
 
 1321#if CURLVERSION_AT_LEAST(7,15,5) 
 1322                curl_easy_setopt(worker->
_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)(avg));
 
 1354  WAR << 
"overall result" << endl;
 
 1355  for (
auto workeriter = 
_workers.begin(); workeriter != 
_workers.end(); ++workeriter)
 
 
 1375  MIL << 
"MediaMultiCurl::MediaMultiCurl(" << url_r << 
", " << attach_point_hint_r << 
")" << endl;
 
 
 1389      curl_multi_cleanup(
_multi);
 
 1392  std::map<std::string, CURL *>::iterator it;
 
 1395      CURL *easy = it->second;
 
 1398          curl_easy_cleanup(easy);
 
 
 1414  for (; sl; sl = sl->next)
 
 
 1432  if ( curl_easy_getinfo( 
_curl, CURLINFO_PRIVATE, &fp ) != CURLE_OK || !fp )
 
 1434  if ( ftell( fp ) == 0 )
 
 1439  long httpReturnCode = 0;
 
 1440  if (curl_easy_getinfo(
_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0)
 
 1444  bool ismetalink = 
false;
 
 1445  if (curl_easy_getinfo(
_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
 
 1447      std::string ct = std::string(ptr);
 
 1448      if (ct.find(
"application/x-zsync") == 0 || ct.find(
"application/metalink+xml") == 0 || ct.find(
"application/metalink4+xml") == 0)
 
 1451  if (!ismetalink && dlnow < 256)
 
 1460      DBG << 
"looks_like_meta_file: " << ismetalink << endl;
 
 
 1477  if( assert_dir( dest.
dirname() ) )
 
 1479    DBG << 
"assert_dir " << dest.
dirname() << 
" failed" << endl;
 
 1489      ERR << 
"out of memory for temp file name" << endl;
 
 1493    AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
 
 1496      ERR << 
"mkstemp failed for file '" << destNew << 
"'" << endl;
 
 1501    file = ::fdopen( tmp_fd, 
"we" );
 
 1504      ERR << 
"fopen failed for file '" << destNew << 
"'" << endl;
 
 1507    tmp_fd.resetDispose();      
 
 1510  DBG << 
"dest: " << dest << endl;
 
 1511  DBG << 
"temp: " << destNew << endl;
 
 1516    curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
 
 1521    curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
 
 1522    curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
 
 1528  curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (*file) );      
 
 1535      curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
 
 1536      curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
 
 1538      curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
 
 1541  curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
 
 1542  curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
 
 1544  curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
 
 1545  long httpReturnCode = 0;
 
 1546  CURLcode infoRet = curl_easy_getinfo(
_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode);
 
 1547  if (infoRet == CURLE_OK)
 
 1550    if ( httpReturnCode == 304
 
 1551         || ( httpReturnCode == 213 && 
_url.getScheme() == 
"ftp" ) ) 
 
 1553      DBG << 
"not modified: " << 
PathInfo(dest) << endl;
 
 1559    WAR << 
"Could not get the response code." << endl;
 
 1565  if (curl_easy_getinfo(
_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
 
 1567      std::string ct = std::string(ptr);
 
 1568      if (ct.find(
"application/x-zsync") == 0 )
 
 1570      else if (ct.find(
"application/metalink+xml") == 0 || ct.find(
"application/metalink4+xml") == 0)
 
 1584      bool userabort = 
false;
 
 1590          std::vector<Url> urls;
 
 1594            bl = 
parser.getBlockList();
 
 1610            XXX << 
"With no blocks" << std::endl;
 
 1617            XXX << 
"No filesize in metalink file and no expected filesize, aborting multicurl." << std::endl;
 
 1622          Disabling 
this workaround 
for now, since we now 
do zip ranges into bigger requests
 
 1633          file = fopen((*destNew).c_str(), 
"w+e");
 
 1638              XXX << 
"reusing blocks from file " << 
target << endl;
 
 1644              XXX << 
"reusing blocks from file " << failedFile << endl;
 
 1652              XXX << 
"reusing blocks from file " << df << endl;
 
 1662              userabort = ex.
errstr() == 
"User abort";
 
 1673          WAR<< 
"Failed to multifetch file " << ex << 
" falling back to single Curl download!" << std::endl;
 
 1674          if (
PathInfo(destNew).size() >= 63336)
 
 1676              ::unlink(failedFile.
asString().c_str());
 
 1683          file = fopen((*destNew).c_str(), 
"w+e");
 
 1695      ERR << 
"Failed to chmod file " << destNew << endl;
 
 1702      ERR << 
"Fclose failed for file '" << destNew << 
"'" << endl;
 
 1706  if ( rename( destNew, dest ) != 0 )
 
 1708      ERR << 
"Rename failed" << endl;
 
 1711  destNew.resetDispose();       
 
 
 1719  if (filesize == off_t(-1) && blklist.haveFilesize())
 
 1720    filesize = blklist.getFilesize();
 
 1721  if (!blklist.haveBlocks() && filesize != 0) {
 
 1722    if ( filesize == -1 ) {
 
 1727    MIL << 
"Generate blocklist, since there was none in the metalink file." << std::endl;
 
 1732    while ( currOff <  filesize )  {
 
 1734      auto blksize = filesize - currOff ;
 
 1735      if ( blksize > prefSize )
 
 1738      blklist.addBlock( currOff, blksize );
 
 1742    XXX << 
"Generated blocklist: " << std::endl << blklist << std::endl << 
" End blocklist " << std::endl;
 
 1745  if (filesize == 0 || !blklist.numBlocks()) {
 
 1754      _multi = curl_multi_init();
 
 1760  std::vector<Url> myurllist;
 
 1761  for (std::vector<Url>::iterator urliter = urllist->begin(); urliter != urllist->end(); ++urliter)
 
 1765          std::string scheme = urliter->getScheme();
 
 1766          if (scheme == 
"http" || scheme == 
"https" || scheme == 
"ftp" || scheme == 
"tftp")
 
 1776  if (!myurllist.size())
 
 1777    myurllist.push_back(baseurl);
 
 
 1786  if (fseeko(fp, off_t(0), SEEK_SET))
 
 1792  while ((l = fread(buf, 1, 
sizeof(buf), fp)) > 0)
 
 
 1800  return _dnsok.find(host) == 
_dnsok.end() ? false : 
true;
 
 
 1822    curl_easy_cleanup(oldeasy);
 
 
void resetDispose()
Set no dispose function.
Store and operate with byte count.
static const Unit MB
1000^2 Byte
static const Unit K
1024 Byte
std::string asString(unsigned field_width_r=0, unsigned unit_width_r=1) const
Auto selected Unit and precision.
Compute Message Digests (MD5, SHA1 etc)
bool update(const char *bytes, size_t len)
feed data into digest computation algorithm
Digest clone() const
Returns a clone of the current Digest and returns it.
Base class for Exception.
std::string asString() const
Returns a default string representation of the Url object.
Pathname repoCachePath() const
Path where the caches are kept (/var/cache/zypp)
static ZConfig & instance()
Singleton ctor.
Wrapper class for stat/lstat.
Pathname dirname() const
Return all but the last component od this path.
const std::string & asString() const
String representation.
The CurlMultiPartHandler class.
zypp::Url propagateQueryParams(zypp::Url url_r, const zypp::Url &template_r)
String related utilities and Regular expression matching.
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
int unlink(const Pathname &path)
Like 'unlink'.
std::string numstring(char n, int w=0)
int zypp_poll(std::vector< GPollFD > &fds, int timeout)
Small wrapper around g_poll that additionally listens to the shutdown FD returned by ZYpp::shutdownSi...
Easy-to use interface to the ZYPP dependency resolver.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
auto eintrSafeCall(Fun &&function, Args &&... args)
CURLMcode handleSocketActions(const std::vector< GPollFD > &actionsFds, int first=0)
std::vector< GPollFD > socks
std::optional< long > timeout_ms
AutoDispose<int> calling ::close
AutoDispose<FILE*> calling ::fclose
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.