445    DBG << 
"Delta XFER: Can not reuse blocks because we have no chksumlen" << std::endl;
 
  449  if ( (fp = fopen(filename.c_str(), 
"r")) == 0 ) {
 
  450    DBG << 
"Delta XFER: Can not reuse blocks, unable to open file "<< filename << std::endl;
 
  454  size_t nblks = 
blocks.size();
 
  455  std::vector<bool> found( nblks + 1 );
 
  458      const auto rsumAMask = 
rsumlen < 3 ? 0 : 
rsumlen == 3 ? 0xff : 0xffff;
 
  464      auto zsyncRsumsData = std::make_unique<rsum[]>( 
rsums.size() + 
rsumseq );
 
  465      auto zsyncRsums = zsyncRsumsData.get();
 
  466      for ( std::size_t i = 0; i < 
rsums.size(); i++ ) {
 
  467        const auto &rs = 
rsums[i];
 
  468        unsigned short s = 0, m = 0;
 
  469        s = (rs >> 16) & 65535;
 
  471        zsyncRsums[i] = rsum{ s, m };
 
  475      const auto & calc_rhash = [&]( 
const rsum* e ) -> 
unsigned {
 
  478          for ( uint i = 1; i < 
rsumseq; i++ ) {
 
  482          h ^= ( e[0].a & rsumAMask ) << 3;
 
  487      size_t blksize = 
blocks[0].size;
 
  493      uint rsumHashMask = 0;
 
  498        while ((2U << (i - 1)) > nblks && i > 4)
 
  502        rsumHashMask = (2U << i) - 1;
 
  506      auto rsumHashTableData = std::make_unique<std::vector<size_t>[]>( rsumHashMask + 1 );
 
  507      auto rsumHashTable = rsumHashTableData.get();
 
  510      for ( 
size_t id = 0; 
id < nblks; 
id++) {
 
  511        const auto hash = calc_rhash( &zsyncRsums[
id] );
 
  512        auto &hashList = rsumHashTable[ hash & rsumHashMask ];
 
  513        hashList.push_back(
id);
 
  517      constexpr auto BLOCKCNT = 16;
 
  521      const auto readBufSize = blksize * 
rsumseq * BLOCKCNT;
 
  524      auto readBufData = std::make_unique<unsigned char[]>( readBufSize );
 
  525      memset(readBufData.get(), 0, blksize);
 
  528      auto readBuf = readBufData.get();
 
  531      auto seqRsumsData = std::make_unique<rsum[]> ( 
rsumseq );
 
  532      auto seqRsums = seqRsumsData.get();
 
  537      if ((blksize & (blksize - 1)) == 0)
 
  538        for (bshift = 0; size_t(1 << bshift) != blksize; bshift++)
 
  544      std::optional<size_t> nextReqMatchInSequence;
 
  545      off_t dataOffset = 0; 
 
  549      const auto &tryWriteMatchingBlocks  = [&]( 
const std::vector<size_t> &list, 
const u_char *currBuf, uint reqMatches ){
 
  551        int targetBlocksWritten = 0;
 
  554        nextReqMatchInSequence.reset();
 
  556        for ( 
const auto blkno : list ) {
 
  561          const auto blockRsum = &zsyncRsums[blkno];
 
  563          uint weakMatches = 0;
 
  567          if ( (seqRsums[0].
a & rsumAMask) != blockRsum[0].
a ||
 
  568               seqRsums[0].
b != blockRsum[0].
b )
 
  573          for ( uint i = 1; i < reqMatches; i++ ) {
 
  574            if ( (seqRsums[i].
a & rsumAMask) != blockRsum[i].
a ||
 
  575                 seqRsums[i].
b != blockRsum[i].
b )
 
  580          if ( weakMatches < reqMatches )
 
  584          uint realMatches = 0;
 
  585          for( uint i = 0; i < reqMatches; i++ ) {
 
  586            if ( !
checkChecksum(blkno + i, currBuf + ( i * blksize ), blksize ) ) {
 
  593          if( realMatches < reqMatches )
 
  598          const auto nextPossibleMatch = blkno + realMatches;
 
  599          if ( !found[nextPossibleMatch] )
 
  600            nextReqMatchInSequence = nextPossibleMatch; 
 
  602          for( uint i = 0; i < realMatches; i++ ) {
 
  603            writeBlock( blkno + i, wfp, currBuf + ( i * blksize ), blksize, 0, found );
 
  604            targetBlocksWritten++;
 
  607        return targetBlocksWritten;
 
  613      const off_t seqMatchLen = ( blksize * 
rsumseq ); 
 
  615      while (! feof(fp) ) {
 
  618            dataLen = fread( readBuf, 1, readBufSize, fp );
 
  622            const auto remainLen = dataLen-dataOffset;
 
  624              memmove( readBuf, readBuf+dataOffset, remainLen );
 
  626            dataLen = fread( readBuf+remainLen, 1, readBufSize-remainLen, fp );
 
  627            dataLen += remainLen;
 
  633            memset( readBuf + dataLen, 0, readBufSize - dataLen );
 
  634            dataLen = readBufSize;
 
  637          if ( dataLen < seqMatchLen )
 
  641          for( uint i = 0; i < 
rsumseq; i++ )
 
  642            seqRsums[i] = rcksum_calc_rsum_block( readBuf + ( i * blksize ), blksize );
 
  647            if ( dataOffset + seqMatchLen > dataLen )
 
  650            u_char *currBuf = readBuf + dataOffset;
 
  654            uint deltaBlocksMatched = 0;
 
  656            if ( nextReqMatchInSequence.has_value() ) {
 
  657              if ( tryWriteMatchingBlocks( { *nextReqMatchInSequence }, currBuf, 1 ) > 0 )
 
  658                deltaBlocksMatched = 1;
 
  661              const auto hash = calc_rhash( seqRsums );
 
  664              auto &blockListForHash = rsumHashTable[ hash & rsumHashMask ];
 
  665              if ( blockListForHash.size() ) {
 
  666                if ( tryWriteMatchingBlocks( blockListForHash, currBuf, 
rsumseq ) > 0 )
 
  671            if ( deltaBlocksMatched > 0 ) {
 
  673              dataOffset += ( deltaBlocksMatched * blksize );
 
  675              if ( dataOffset + seqMatchLen > dataLen )
 
  678              if ( deltaBlocksMatched < 
rsumseq ) {
 
  682              for( uint i = 0; i < 
rsumseq; i++ )
 
  683                seqRsums[i] = rcksum_calc_rsum_block( readBuf + dataOffset + ( i * blksize ), blksize );
 
  689              if ( dataOffset + seqMatchLen > dataLen )
 
  691              for ( uint i = 0; i < 
rsumseq; i++ ) {
 
  692                const auto blkOff = ( i*blksize );
 
  693                u_char oldC = (currBuf + blkOff)[0];
 
  694                u_char newC = (currBuf + blkOff)[blksize];
 
  695                UPDATE_RSUM( seqRsums[i].
a, seqRsums[i].
b, oldC, newC, bshift );
 
  706      auto buf = std::make_unique<unsigned char[]>( bufl );
 
  707      for (
size_t blkno = 0; blkno < 
blocks.size(); ++blkno)
 
  709          if (off > 
blocks[blkno].off)
 
  711          size_t blksize = 
blocks[blkno].size;
 
  715              buf = std::make_unique<unsigned char[]>( bufl );
 
  717          size_t skip = 
blocks[blkno].off - off;
 
  720              size_t l = skip > bufl ? bufl : skip;
 
  721              if (fread(buf.get(), l, 1, fp) != 1)
 
  726          if (fread(buf.get(), blksize, 1, fp) != 1)
 
  729            writeBlock(blkno, wfp, buf.get(), blksize, 0, found);
 
  734    DBG << 
"Delta XFER: No reusable blocks found for " << filename << std::endl;
 
  738  std::vector<MediaBlock> nblocks;
 
  739  std::vector<unsigned char> nchksums;
 
  740  std::vector<unsigned int> nrsums;
 
  742  size_t originalSize = 0;
 
  744  for (
size_t blkno = 0; blkno < 
blocks.size(); ++blkno)
 
  746      const auto &blk = 
blocks[blkno];
 
  747      originalSize += blk.size;
 
  751          nblocks.push_back(blk);
 
  755              nchksums.resize(nblocks.size() * 
chksumlen);
 
  759            nrsums.push_back(
rsums[blkno]);
 
  762  DBG << 
"Delta XFER: Found blocks to reuse, " << 
blocks.size() << 
" vs " << nblocks.size() << 
", resused blocks: " << 
blocks.size() - nblocks.size() << 
"\n" 
  763      << 
"Old transfer size: " << originalSize << 
" new size: " << newSize << std::endl;
 
 
  775    DBG << 
"Delta XFER: Can not reuse blocks because we have no chksumlen" << std::endl;
 
  779  if ( (fp = fopen(filename.c_str(), 
"r")) == 0 ) {
 
  780    DBG << 
"Delta XFER: Can not reuse blocks, unable to open file "<< filename << std::endl;
 
  783  size_t nblks = 
blocks.size();
 
  784  std::vector<bool> found;
 
  785  found.resize(nblks + 1);
 
  788      size_t blksize = 
blocks[0].size;
 
  797      unsigned int hm = 
rsums.size() * 2;
 
  798      while (hm & (hm - 1))  {
 
  808      auto rsumHashTable = std::make_unique<unsigned int[]>( hm+1 );
 
  809      memset(rsumHashTable.get(), 0, (hm + 1) * 
sizeof(
unsigned int));
 
  812      for (
unsigned int i = 0; i < 
rsums.size(); i++)
 
  814          if (
blocks[i].size != blksize && (i != nblks - 1 || 
rsumpad != blksize))
 
  816          unsigned int r = 
rsums[i];
 
  817          unsigned int h = r & hm;
 
  819          while (rsumHashTable[h])
 
  821          rsumHashTable[h] = i + 1;
 
  830      auto ringBuf = std::make_unique<unsigned char[]>( blksize );
 
  833      auto buf2 = std::make_unique<unsigned char[]>( blksize );
 
  839      unsigned char *pushbackp = 0;
 
  844      if ((blksize & (blksize - 1)) == 0)
 
  845        for (bshift = 0; size_t(1 << bshift) != blksize; bshift++)
 
  851      unsigned short a = 0, 
b = 0;
 
  853      memset(ringBuf.get(), 0, blksize);
 
  862          for (
size_t i = 0; i < blksize; i++)
 
  901                b += 
a - ( oc << bshift );
 
  909                  if (
size_t(i) != blksize - 1)
 
  914              unsigned int r = ((
unsigned int)
a & 65535) << 16 | ((
unsigned int)
b & 65535);
 
  917              unsigned int h = r & hm;
 
  921              for (; rsumHashTable[h]; h = (h + hh++) & hm)
 
  923                  size_t blkno = rsumHashTable[h] - 1;
 
  926                  if (
rsums[blkno] != r)
 
  935                      if (eof || blkno + 1 >= nblks)
 
  937                      pushback = 
fetchnext(fp, buf2.get(), blksize, pushback, pushbackp);
 
  938                      pushbackp = buf2.get();
 
  942                      if (!
checkRsum(blkno + 1, buf2.get(), blksize))
 
  955                  writeBlock(blkno, wfp, ringBuf.get(), blksize, i + 1, found);
 
  958                      writeBlock(blkno + 1, wfp, buf2.get(), blksize, 0, found);
 
  967                      pushback = 
fetchnext(fp, buf2.get(), blksize, pushback, pushbackp);
 
  968                      pushbackp = buf2.get();
 
  972                      if (!
checkRsum(blkno, buf2.get(), blksize))
 
  978                      writeBlock(blkno, wfp, buf2.get(), blksize, 0, found);
 
  985                  memset(ringBuf.get(), 0, blksize);
 
  998      auto buf = std::make_unique<unsigned char[]>( bufl );
 
  999      for (
size_t blkno = 0; blkno < 
blocks.size(); ++blkno)
 
 1001          if (off > 
blocks[blkno].off)
 
 1003          size_t blksize = 
blocks[blkno].size;
 
 1007              buf = std::make_unique<unsigned char[]>( bufl );
 
 1009          size_t skip = 
blocks[blkno].off - off;
 
 1012              size_t l = skip > bufl ? bufl : skip;
 
 1013              if (fread(buf.get(), l, 1, fp) != 1)
 
 1018          if (fread(buf.get(), blksize, 1, fp) != 1)
 
 1021            writeBlock(blkno, wfp, buf.get(), blksize, 0, found);
 
 1025  if (!found[nblks]) {
 
 1026    DBG << 
"Delta XFER: No reusable blocks found for " << filename << std::endl;
 
 1030  std::vector<MediaBlock> nblocks;
 
 1031  std::vector<unsigned char> nchksums;
 
 1032  std::vector<unsigned int> nrsums;
 
 1034  size_t originalSize = 0;
 
 1036  for (
size_t blkno = 0; blkno < 
blocks.size(); ++blkno)
 
 1038      const auto &blk = 
blocks[blkno];
 
 1039      originalSize += blk.size;
 
 1043          nblocks.push_back(blk);
 
 1044          newSize += blk.size;
 
 1047              nchksums.resize(nblocks.size() * 
chksumlen);
 
 1051            nrsums.push_back(
rsums[blkno]);
 
 1054  DBG << 
"Delta XFER: Found blocks to reuse, " << 
blocks.size() << 
" vs " << nblocks.size() << 
", resused blocks: " << 
blocks.size() - nblocks.size() << 
"\n" 
 1055      << 
"Old transfer size: " << originalSize << 
" new size: " << newSize << std::endl;