21#include <zypp-core/base/DefaultIntegral> 
   23#include <zypp-media/MediaException> 
   27#include <zypp-core/base/UserRequestException> 
   31#undef ZYPP_BASE_LOGGER_LOGGROUP 
   32#define ZYPP_BASE_LOGGER_LOGGROUP "zypp::fetcher" 
   68      if ( lhs->location.medianr() == rhs->location.medianr() )
 
   69        return lhs->location.filename() < rhs->location.filename();
 
   71      return lhs->location.medianr() < rhs->location.medianr();
 
 
 
  118    return str << obj->location;
 
 
  145        return a._pathName < 
b._pathName;
 
 
 
  150    Fetcher::Options 
options() 
const;
 
  213      void validate( 
const Pathname & localfile_r, 
const std::list<FileChecker> & checkers_r );
 
  219                        const Pathname &dest_dir, FetcherJob::Flags flags );
 
  237    { 
return new Impl( *
this ); }
 
 
  240    std::set<FetcherIndex_Ptr,SameFetcherIndex> 
_indexes;
 
 
  282        job->checkers.push_back(checker);
 
 
  297        job->checkers.push_back(checker);
 
 
  315      job->checkers.push_back(checker);
 
 
  321    MIL << 
"adding index " << resource << endl;
 
 
  346        DBG << 
"Adding fetcher cache: '" << cache_dir << 
"'." << endl;
 
  352        ERR << 
"Not adding cache: '" << cache_dir << 
"'. Not a directory." << endl;
 
  357        ERR << 
"Not adding cache '" << cache_dir << 
"'. Path does not exists." << endl;
 
 
  370    if ( 
PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.
checksum() ) ) {
 
  371      return cacheLocation;
 
  374    MIL << 
"start fetcher with " << 
_caches.size() << 
" cache directories." << endl;
 
  377      if ( 
PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.
checksum() ) )
 
  379        MIL << 
"file " << resource_r.
filename() << 
" found in cache " << cacheInfo._pathName << endl;
 
  382        return cacheLocation;
 
 
  393      MIL << 
"Checking job [" << localfile_r << 
"] (" << checkers_r.size() << 
" checkers )" << endl;
 
  398          chkfnc( localfile_r );
 
  400          ERR << 
"Invalid checker for '" << localfile_r << 
"'" << endl;
 
 
  423      auto fnc_addIfInContent( [&]( 
const std::string & index_r ) -> 
bool 
  438        fnc_addIfInContent( 
"CHECKSUMS" ) || fnc_addIfInContent( 
"SHA1SUMS" );
 
  442        fnc_addIfInContent( 
"content" );
 
 
  455          std::copy(filled.begin(), filled.end(), std::back_inserter(content));
 
  460          media.dirInfo( tofill,
 
  464          std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
 
 
  471                                  const Pathname &dest_dir, FetcherJob::Flags flags  )
 
  475      MIL << 
"Adding directory " << resource.
filename() << endl;
 
  483        WAR << 
"Skipping subtree hidden at " << resource.
filename() << endl;
 
  491      for ( filesystem::DirContent::const_iterator it = content.begin();
 
  514                  WAR << 
"Resource " << filename << 
" has no checksum in the index either." << endl;
 
 
  545      if ( tmpFile.
empty() )
 
  547        MIL << 
"Not found in cache, retrieving..." << endl;
 
  556      if ( tmpFile == destFullPath )
 
  560      validate( tmpFile, jobp_r->checkers );
 
  563      if ( tmpFile == destFullPath )
 
  567        if ( assert_dir( destFullPath->
dirname() ) != 0 )
 
  579        WAR << 
"optional resource " << resource << 
" could not be transferred." << endl;
 
 
  609    else if ( index.
basename() == 
"content" )
 
  612      WAR << index << 
": index file format not known" << endl;
 
 
  620      MIL << index << 
" contains " << reader.
_repoindex->mediaFileChecksums.size() << 
" checksums." << endl;
 
 
  631      std::ifstream in( index.
c_str() );
 
  635          while ( getline( in, buffer ) )
 
  638              if ( buffer[0] == 
'#' )
 
  642              if ( checksum.empty() )
 
  645              if ( buffer.empty() )
 
  647                WAR << 
"Missing filename in CHECKSUMS file: " << index.
asString() << 
" (" << checksum << 
")" << endl;
 
 
  660    MIL << 
"downloading index " << resource << endl;
 
  700        getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.
filename()), 
false);
 
  702        WAR << 
"No public key specified by user for index '" << keyloc.
filename() << 
"'"<< endl;
 
 
  719          MIL << 
"No indexes to read." << endl;
 
  725        if ( (*it_idx)->read )
 
  727          DBG << 
"Already read index " << 
PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
 
  734          readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().
dirname() );
 
  736          MIL << 
"Remember read index " << 
PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
 
  737          (*it_idx)->read = 
true;
 
  740      MIL << 
"done reading indexes" << endl;
 
 
  760    progress.
sendTo(progress_receiver);
 
  786          MIL << 
"Autodiscovering signed indexes on '" 
  787              << jobp->location.filename().dirname() << 
"' for '" 
  788              << jobp->location.filename() << 
"'" << endl;
 
  797          MIL << 
"Autodiscovering signed indexes on '" 
  799              << jobp->location.filename() << 
"'" << endl;
 
  806      if ( jobp->location.checksum().empty() )
 
  808          if ( 
_checksums.find(jobp->location.filename().asString())
 
  813              jobp->checkers.push_back(digest_check);
 
  823                  jobp->checkers.push_back(digest_check);
 
  831          jobp->checkers.push_back(digest_check);
 
  838      if ( ! progress.
incr() )
 
 
  846      for ( std::list<FetcherJob_Ptr>::const_iterator it_res = obj.
_resources.begin(); it_res != obj.
_resources.end(); ++it_res )
 
 
  877    _pimpl->enqueueDigested( resource, checker );
 
 
  884      _pimpl->enqueueDir(resource, recursive, checker);
 
 
  891      _pimpl->enqueueDigestedDir(resource, recursive, checker);
 
 
  897    _pimpl->addIndex(resource);
 
 
  903    _pimpl->enqueue(resource, checker);
 
 
  929    _pimpl->start( dest_dir, progress );
 
 
void resetDispose()
Set no dispose function.
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
Integral type with defined initial value when default constructed.
Base class for Exception.
void remember(const Exception &old_r)
Store an other Exception as history.
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
Impl & operator=(const Impl &)=delete
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
std::map< std::string, CheckSum > _checksums
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
ManagedFile locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory).
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
Impl * clone() const
clone for RWCOW_pointer
std::list< FetcherJob_Ptr > _resources
friend Impl * rwcowClone(const Impl *rhs)
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Impl & operator=(Impl &&)=delete
void addIndex(const OnMediaLocation &resource)
std::map< std::string, filesystem::DirContent > _dircontent
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
MediaSetAccess * _mediaSetAccess
Impl(const Impl &)=default
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
Fetcher::Options options() const
std::set< CacheInfo > _caches
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
static shared_ptr< Impl > nullimpl()
Offer default Impl.
void addCachePath(const Pathname &cache_dir, Fetcher::CacheOptions options)
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress)
void setOptions(Fetcher::Options options)
Fetcher::Options _options
void setMediaSetAccess(MediaSetAccess &media)
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
Options options() const
Get current options.
void reset()
Reset the transfer (jobs) list.
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
void setOptions(Options options)
Set the Fetcher options.
@ AutoAddChecksumsIndexes
If a CHECKSUMS file is found, it is downloaded and read.
@ AutoAddContentFileIndexes
If a content file is found, it is downloaded and read.
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
void setMediaSetAccess(MediaSetAccess &media)
Sets the media set access that will be used to precache and to download the files when start is calle...
Maintain [min,max] and counter (value) for progress counting.
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
bool incr(value_type val_r=1)
Increment counter value (default by 1).
Checks for the validity of a signature.
ZYpp::Ptr getZYpp()
Convenience to get the Pointer to the ZYpp instance.
Wrapper class for stat/lstat.
bool isExist() const
Return whether valid stat info exists.
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Pathname dirname() const
Return all but the last component od this path.
const char * c_str() const
String representation.
const std::string & asString() const
String representation.
std::string basename() const
Return the last component of this path.
bool empty() const
Test for an empty path.
Parse repoindex part from a content file.
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
void setRepoIndexConsumer(const RepoIndexConsumer &fnc_r)
Consumer to call when repo index was parsed.
String related utilities and Regular expression matching.
std::list< DirEntry > DirContent
Returned by readdir.
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
int unlink(const Pathname &path)
Like 'unlink'.
std::string stripFirstWord(std::string &line, const bool ltrim_first)
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Easy-to use interface to the ZYPP dependency resolver.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
std::ostream & operator<<(std::ostream &str, const SerialNumber &obj)
shared_ptr< FetcherIndex > FetcherIndex_Ptr
std::string asString(const Patch::Category &obj)
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
shared_ptr< FetcherJob > FetcherJob_Ptr
parser::susetags::RepoIndex_Ptr _repoindex
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
OnMediaLocation location
Index localtion.
FetcherIndex(const OnMediaLocation &loc)
DefaultIntegral< bool, false > read
Whether we read this index.
Class to encapsulate the OnMediaLocation object and the FileChecker together.
std::list< FileChecker > checkers
FetcherJob(const FetcherJob &)=default
FetcherJob(FetcherJob &&)=default
FetcherJob & operator=(const FetcherJob &)=default
ZYPP_DECLARE_FLAGS(Flags, Flag)
FetcherJob & operator=(FetcherJob &&)=default
FetcherJob(const OnMediaLocation &loc)
Fetcher::CacheOptions _options
friend bool operator<(const CacheInfo &a, const CacheInfo &b)
std::set ordering (less semantic)
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs) const
Listentry returned by readdir.
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
#define ZYPP_DECLARE_OPERATORS_FOR_FLAGS(Name)