libzypp  17.35.16
repodownloaderwf.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
9 #include "repodownloaderwf.h"
12 
13 #include <utility>
14 #include <zypp-media/ng/Provide>
15 #include <zypp-media/ng/ProvideSpec>
16 #include <zypp/ng/Context>
17 #include <zypp/ng/repo/Downloader>
18 #include <zypp/PublicKey.h>
19 #include <zypp/KeyRing.h>
20 
25 
26 // sync workflow helpers
29 
30 #undef ZYPP_BASE_LOGGER_LOGGROUP
31 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp::repomanager"
32 
33 namespace zyppng {
34  namespace {
35 
36  using namespace zyppng::operators;
37 
38  template < class Executor, class OpType >
39  struct DownloadMasterIndexLogic : public LogicBase<Executor, OpType>
40  {
41  public:
42  ZYPP_ENABLE_LOGIC_BASE(Executor, OpType);
43 
44  using DlContextRefType = std::conditional_t<zyppng::detail::is_async_op_v<OpType>, repo::AsyncDownloadContextRef, repo::SyncDownloadContextRef>;
45  using ZyppContextType = typename remove_smart_ptr_t<DlContextRefType>::ContextType;
46  using ProvideType = typename ZyppContextType::ProvideType;
47  using MediaHandle = typename ProvideType::MediaHandle;
48  using ProvideRes = typename ProvideType::Res;
49 
50  DownloadMasterIndexLogic( DlContextRefType &&ctxRef, MediaHandle &&mediaHandle, zypp::filesystem::Pathname &&masterIndex_r )
51  : _dlContext( std::move(ctxRef) )
52  , _media(std::move( mediaHandle ))
53  , _masterIndex(std::move( masterIndex_r ))
54  { }
55 
56  public:
57  MaybeAsyncRef<expected<DlContextRefType>> execute( ) {
58  // always download them, even if repoGpgCheck is disabled
59  _sigpath = _masterIndex.extend( ".asc" );
60  _keypath = _masterIndex.extend( ".key" );
61  _destdir = _dlContext->destDir();
62 
63  auto providerRef = _dlContext->zyppContext()->provider();
64  return std::vector {
65  // fetch signature and keys
66  providerRef->provide( _media, _sigpath, ProvideFileSpec().setOptional( true ).setDownloadSize( zypp::ByteCount( 20, zypp::ByteCount::MB ) ) )
67  | and_then( ProvideType::copyResultToDest ( providerRef, _destdir / _sigpath ) ),
68  providerRef->provide( _media, _keypath, ProvideFileSpec().setOptional( true ).setDownloadSize( zypp::ByteCount( 20, zypp::ByteCount::MB ) ) )
69  | and_then( ProvideType::copyResultToDest ( providerRef, _destdir / _keypath ) ),
70  }
71  | join()
72  | [this]( std::vector<expected<zypp::ManagedFile>> &&res ) {
73 
74  // remember downloaded files
75  std::for_each( res.begin (), res.end(),
76  [this]( expected<zypp::ManagedFile> &f){
77  if (f.is_valid () ) {
78  _dlContext->files().push_back( std::move(f.get()));
79  }
80  });
81 
82  // get the master index file
83  return provider()->provide( _media, _masterIndex, ProvideFileSpec().setDownloadSize( zypp::ByteCount( 20, zypp::ByteCount::MB ) ) );
84  }
85  // execute plugin verification if there is one
86  | and_then( std::bind( &DownloadMasterIndexLogic::pluginVerification, this, std::placeholders::_1 ) )
87 
88  // signature checking
89  | and_then( std::bind( &DownloadMasterIndexLogic::signatureCheck, this, std::placeholders::_1 ) )
90 
91  // copy everything into a directory
92  | and_then( ProvideType::copyResultToDest ( providerRef, _destdir / _masterIndex ) )
93 
94  // final tasks
95  | and_then([this]( zypp::ManagedFile &&masterIndex ) {
96  // Accepted!
97  _dlContext->repoInfo().setMetadataPath( _destdir );
98  _dlContext->repoInfo().setValidRepoSignature( _repoSigValidated );
99 
100  // release the media handle
101  _media = MediaHandle();
102  auto &allFiles = _dlContext->files();
103 
104  // make sure the masterIndex is in front
105  allFiles.insert( allFiles.begin (), std::move(masterIndex) );
106  return make_expected_success( std::move(_dlContext) );
107  });
108  }
109 
110 
111  private:
112  auto provider () {
113  return _dlContext->zyppContext()->provider();
114  }
115 
116  MaybeAsyncRef<expected<ProvideRes>> signatureCheck ( ProvideRes &&res ) {
117 
118  if ( _dlContext->repoInfo().repoGpgCheck() ) {
119 
120  // The local files are in destdir_r, if they were present on the server
121  zypp::Pathname sigpathLocal { _destdir/_sigpath };
122  zypp::Pathname keypathLocal { _destdir/_keypath };
123  bool isSigned = zypp::PathInfo(sigpathLocal).isExist();
124 
125  if ( isSigned || _dlContext->repoInfo().repoGpgCheckIsMandatory() ) {
126 
127  auto verifyCtx = zypp::keyring::VerifyFileContext( res.file() );
128 
129  // only add the signature if it exists
130  if ( isSigned )
131  verifyCtx.signature( sigpathLocal );
132 
133  // only add the key if it exists
134  if ( zypp::PathInfo(keypathLocal).isExist() ) {
135  try {
136  _dlContext->zyppContext()->keyRing()->importKey( zypp::PublicKey(keypathLocal), false );
137  } catch (...) {
139  }
140  }
141 
142  // set the checker context even if the key is not known
143  // (unsigned repo, key file missing; bnc #495977)
144  verifyCtx.keyContext( _dlContext->repoInfo() );
145 
146  return getExtraKeysInRepomd( std::move(res ) )
147  | and_then([this, vCtx = std::move(verifyCtx) ]( ProvideRes &&res ) mutable {
148  for ( const auto &keyData : _buddyKeys ) {
149  DBG << "Keyhint remember buddy " << keyData << std::endl;
150  vCtx.addBuddyKey( keyData.id() );
151  }
152 
153  return SignatureFileCheckWorkflow::verifySignature( _dlContext->zyppContext(), std::move(vCtx))
154  | and_then([ this, res = std::move(res) ]( zypp::keyring::VerifyFileContext verRes ){
155  // remember the validation status
156  _repoSigValidated = verRes.fileValidated();
157  return make_expected_success(std::move(res));
158  });
159  });
160 
161  } else {
162  WAR << "Accept unsigned repository because repoGpgCheck is not mandatory for " << _dlContext->repoInfo().alias() << std::endl;
163  }
164  } else {
165  WAR << "Signature checking disabled in config of repository " << _dlContext->repoInfo().alias() << std::endl;
166  }
168  }
169 
170  // execute the repo verification if there is one
171  expected<ProvideRes> pluginVerification ( ProvideRes &&prevRes ) {
172  // The local files are in destdir_r, if they were present on the server
173  zypp::Pathname sigpathLocal { _destdir/_sigpath };
174  zypp::Pathname keypathLocal { _destdir/_keypath };
175  if ( _dlContext->pluginRepoverification() && _dlContext->pluginRepoverification()->isNeeded() ) {
176  try {
177  _dlContext->pluginRepoverification()->getChecker( sigpathLocal, keypathLocal, _dlContext->repoInfo() )( prevRes.file() );
178  } catch ( ... ) {
179  return expected<ProvideRes>::error( std::current_exception () );
180  }
181  }
182  return make_expected_success(std::move(prevRes));
183  }
184 
189  MaybeAsyncRef<expected<ProvideRes>> getExtraKeysInRepomd ( ProvideRes &&res ) {
190 
191  if ( _masterIndex.basename() != "repomd.xml" ) {
192  return makeReadyResult( expected<ProvideRes>::success( std::move(res) ) );
193  }
194 
195  std::vector<std::pair<std::string,std::string>> keyhints { zypp::parser::yum::RepomdFileReader(res.file()).keyhints() };
196  if ( keyhints.empty() )
197  return makeReadyResult( expected<ProvideRes>::success( std::move(res) ) );
198  DBG << "Check keyhints: " << keyhints.size() << std::endl;
199 
200  auto keyRing { _dlContext->zyppContext()->keyRing() };
201  return zypp::parser::yum::RepomdFileReader(res.file()).keyhints()
202  | transform([this, keyRing]( std::pair<std::string, std::string> val ) {
203 
204  const auto& [ file, keyid ] = val;
205  auto keyData = keyRing->trustedPublicKeyData( keyid );
206  if ( keyData ) {
207  DBG << "Keyhint is already trusted: " << keyid << " (" << file << ")" << std::endl;
208  return makeReadyResult ( expected<zypp::PublicKeyData>::success(keyData) ); // already a trusted key
209  }
210 
211  DBG << "Keyhint search key " << keyid << " (" << file << ")" << std::endl;
212 
213  keyData = keyRing->publicKeyData( keyid );
214  if ( keyData )
216 
217  // TODO: Enhance the key caching in general...
218  const zypp::ZConfig & conf = _dlContext->zyppContext()->config();
219  zypp::Pathname cacheFile = conf.repoManagerRoot() / conf.pubkeyCachePath() / file;
220 
221  return zypp::PublicKey::noThrow(cacheFile)
222  | [ keyid = keyid ]( auto &&key ){
223  if ( key.fileProvidesKey( keyid ) )
224  return make_expected_success( std::forward<decltype(key)>(key) );
225  else
226  return expected<zypp::PublicKey>::error( std::make_exception_ptr (zypp::Exception("File does not provide key")));
227  }
228  | or_else ([ this, file = file, keyid = keyid, cacheFile ] ( auto ) mutable -> MaybeAsyncRef<expected<zypp::PublicKey>> {
229  auto providerRef = _dlContext->zyppContext()->provider();
230  return providerRef->provide( _media, file, ProvideFileSpec().setOptional(true) )
231  | and_then( ProvideType::copyResultToDest( providerRef, _destdir / file ) )
232  | and_then( [this, providerRef, file, keyid , cacheFile = std::move(cacheFile)]( zypp::ManagedFile &&res ) {
233 
234  // remember we downloaded the file
235  _dlContext->files().push_back ( std::move(res) );
236 
237  auto key = zypp::PublicKey::noThrow( _dlContext->files().back() );
238  if ( not key.fileProvidesKey( keyid ) ) {
239  const auto &str = zypp::str::Str() << "Keyhint " << file << " does not contain a key with id " << keyid << ". Skipping it.";
240  WAR << str << std::endl;
241  return makeReadyResult(expected<zypp::PublicKey>::error( std::make_exception_ptr( zypp::Exception(str)) ));
242  }
243 
244  // Try to cache it...
245  zypp::filesystem::assert_dir( cacheFile.dirname() );
246  return providerRef->copyFile( key.path(), cacheFile )
247  | [ key ]( expected<zypp::ManagedFile> res ) mutable {
248  if ( res ) {
249  // do not delete from cache
250  res->resetDispose ();
251  }
252  return expected<zypp::PublicKey>::success( std::move(key) );
253  };
254  });
255  })
256  | and_then( [ keyRing, keyid = keyid ]( zypp::PublicKey key ){
257  keyRing->importKey( key, false ); // store in general keyring (not trusted!)
258  return expected<zypp::PublicKeyData>::success(keyRing->publicKeyData( keyid )); // fetch back from keyring in case it was a hidden key
259  });
260  })
261  | [this, res = res] ( std::vector<expected<zypp::PublicKeyData>> &&keyHints ) mutable {
262  std::for_each( keyHints.begin(), keyHints.end(), [this]( expected<zypp::PublicKeyData> &keyData ){
263  if ( keyData && *keyData ) {
264  if ( not zypp::PublicKey::isSafeKeyId( keyData->id() ) ) {
265  WAR << "Keyhint " << keyData->id() << " for " << *keyData << " is not strong enough for auto import. Just caching it." << std::endl;
266  return;
267  }
268  _buddyKeys.push_back ( std::move(keyData.get()) );
269  }
270  });
271 
272  MIL << "Check keyhints done. Buddy keys: " << _buddyKeys.size() << std::endl;
273  return expected<ProvideRes>::success (std::move(res));
274  };
275  }
276 
277  DlContextRefType _dlContext;
278  MediaHandle _media;
280 
284  zypp::TriBool _repoSigValidated = zypp::indeterminate;
285 
286  std::vector<zypp::PublicKeyData> _buddyKeys;
287  };
288 
289  }
290 
292  {
293  return SimpleExecutor<DownloadMasterIndexLogic, AsyncOp<expected<repo::AsyncDownloadContextRef>>>::run( std::move(dl), std::move(mediaHandle), std::move(masterIndex_r) );
294  }
295 
297  {
298  return SimpleExecutor<DownloadMasterIndexLogic, SyncOp<expected<repo::SyncDownloadContextRef>>>::run( std::move(dl), std::move(mediaHandle), std::move(masterIndex_r) );
299  }
300 
302  {
303  using namespace zyppng::operators;
304  return dl->zyppContext()->provider()->attachMediaIfNeeded( mediaHandle )
305  | and_then([ dl, mi = std::move(masterIndex_r) ]( ProvideMediaHandle handle ) mutable {
306  return downloadMasterIndex( std::move(dl), std::move(handle), std::move(mi) );
307  });
308  }
309 
311  {
312  using namespace zyppng::operators;
313  return dl->zyppContext()->provider()->attachMediaIfNeeded( mediaHandle )
314  | and_then([ dl, mi = std::move(masterIndex_r) ]( SyncMediaHandle handle ) mutable {
315  return downloadMasterIndex( std::move(dl), std::move(handle), std::move(mi) );
316  });
317  }
318 
319 
320  namespace {
321  template <class DlContextRefType, class MediaHandleType>
322  auto statusImpl ( DlContextRefType dlCtx, MediaHandleType &&mediaHandle ) {
323 
324  constexpr bool isAsync = std::is_same_v<DlContextRefType,repo::AsyncDownloadContextRef>;
325 
326  const auto finalizeStatus = [ dlCtx ]( zypp::RepoStatus status ){
327  return expected<zypp::RepoStatus>::success( zypp::RepoStatus( dlCtx->repoInfo()) && status );
328  };
329 
330  switch( dlCtx->repoInfo().type().toEnum()) {
332  return RpmmdWorkflows::repoStatus( dlCtx, std::forward<MediaHandleType>(mediaHandle) ) | and_then( std::move(finalizeStatus) );
334  return SuseTagsWorkflows::repoStatus( dlCtx, std::forward<MediaHandleType>(mediaHandle) ) | and_then( std::move(finalizeStatus) );
336  return PlaindirWorkflows::repoStatus ( dlCtx, std::forward<MediaHandleType>(mediaHandle) ) | and_then( std::move(finalizeStatus) );
338  break;
339  }
340 
341  return makeReadyResult<expected<zypp::RepoStatus>, isAsync >( expected<zypp::RepoStatus>::error( ZYPP_EXCPT_PTR (zypp::repo::RepoUnknownTypeException(dlCtx->repoInfo()))) );
342  }
343  }
344 
346  return statusImpl( dl, std::move(mediaHandle) );
347  }
348 
349  expected<zypp::RepoStatus> RepoDownloaderWorkflow::repoStatus(repo::SyncDownloadContextRef dl, SyncMediaHandle mediaHandle) {
350  return statusImpl( dl, std::move(mediaHandle) );
351  }
352 
354  using namespace zyppng::operators;
355  return dl->zyppContext()->provider()->attachMediaIfNeeded( mediaHandle )
356  | and_then([ dl ]( ProvideMediaHandle handle ) {
357  return repoStatus( dl, std::move(handle) );
358  });
359  }
360 
362  using namespace zyppng::operators;
363  return dl->zyppContext()->provider()->attachMediaIfNeeded( mediaHandle )
364  | and_then([ dl ]( SyncMediaHandle handle ) {
365  return repoStatus( dl, std::move(handle) );
366  });
367  }
368 
369 
370  namespace {
371  template <class DlContextRefType, class MediaHandleType>
372  auto downloadImpl ( DlContextRefType dlCtx, MediaHandleType &&mediaHandle, ProgressObserverRef &&progressObserver ) {
373 
374  constexpr bool isAsync = std::is_same_v<DlContextRefType,repo::AsyncDownloadContextRef>;
375 
376  switch( dlCtx->repoInfo().type().toEnum()) {
378  return RpmmdWorkflows::download( std::move(dlCtx), std::forward<MediaHandleType>(mediaHandle), std::move(progressObserver) );
380  return SuseTagsWorkflows::download( std::move(dlCtx), std::forward<MediaHandleType>(mediaHandle), std::move(progressObserver) );
382  return PlaindirWorkflows::download ( std::move(dlCtx), std::forward<MediaHandleType>(mediaHandle) );
384  break;
385  }
386 
387  return makeReadyResult<expected<DlContextRefType>, isAsync >( expected<DlContextRefType>::error( ZYPP_EXCPT_PTR (zypp::repo::RepoUnknownTypeException(dlCtx->repoInfo()))) );
388  }
389  }
390 
391  AsyncOpRef<expected<repo::AsyncDownloadContextRef> > RepoDownloaderWorkflow::download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
392  {
393  return downloadImpl( dl, std::move(mediaHandle), std::move(progressObserver) );
394  }
395 
396  expected<repo::SyncDownloadContextRef> RepoDownloaderWorkflow::download(repo::SyncDownloadContextRef dl, SyncMediaHandle mediaHandle, ProgressObserverRef progressObserver)
397  {
398  return downloadImpl( dl, std::move(mediaHandle), std::move(progressObserver) );
399  }
400 
401  AsyncOpRef<expected<repo::AsyncDownloadContextRef> > RepoDownloaderWorkflow::download(repo::AsyncDownloadContextRef dl, AsyncLazyMediaHandle mediaHandle, ProgressObserverRef progressObserver)
402  {
403  using namespace zyppng::operators;
404  return dl->zyppContext()->provider()->attachMediaIfNeeded( mediaHandle )
405  | and_then([ dl, po = std::move(progressObserver) ]( ProvideMediaHandle handle ) mutable {
406  return downloadImpl( dl, std::move(handle), std::move(po) );
407  });
408  }
409 
410  expected<repo::SyncDownloadContextRef> RepoDownloaderWorkflow::download(repo::SyncDownloadContextRef dl, SyncLazyMediaHandle mediaHandle, ProgressObserverRef progressObserver)
411  {
412  using namespace zyppng::operators;
413  return dl->zyppContext()->provider()->attachMediaIfNeeded( mediaHandle )
414  | and_then([ dl, po = std::move(progressObserver) ]( SyncMediaHandle handle ) mutable {
415  return downloadImpl( dl, std::move(handle), std::move(po) );
416  });
417  }
418 
419 }
#define MIL
Definition: Logger.h:100
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver=nullptr)
boost::logic::tribool TriBool
3-state boolean logic (true, false and indeterminate).
Definition: String.h:30
thrown when it was impossible to determine this repo type.
auto transform(Transformation &&transformation)
Definition: transform.h:70
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:324
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
Definition: susetags.cc:330
Store and operate with byte count.
Definition: ByteCount.h:31
Pathname pubkeyCachePath() const
Path where the pubkey caches.
Definition: ZConfig.cc:1048
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:175
zypp::Pathname _masterIndex
zypp::Pathname _keypath
String related utilities and Regular expression matching.
zypp::TriBool _repoSigValidated
static expected< std::decay_t< Type >, Err > make_expected_success(Type &&t)
Definition: expected.h:397
I/O context for KeyRing::verifyFileSignatureWorkflow.
static const Unit MB
1000^2 Byte
Definition: ByteCount.h:61
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:130
zypp::Pathname _sigpath
#define ZYPP_EXCPT_PTR(EXCPT)
Drops a logline and returns Exception as a std::exception_ptr.
Definition: Exception.h:428
Pathname repoManagerRoot() const
The RepoManager root directory.
Definition: ZConfig.cc:956
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
Definition: plaindir.cc:88
MediaHandle _media
auto or_else(Fun &&function)
Definition: expected.h:630
const Pathname & signature() const
Detached signature or empty.
Convenient building of std::string via std::ostringstream Basically a std::ostringstream autoconverti...
Definition: String.h:211
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:282
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:126
Interim helper class to collect global options and settings.
Definition: ZConfig.h:68
#define WAR
Definition: Logger.h:101
auto join()
Definition: wait.h:133
#define ZYPP_ENABLE_LOGIC_BASE(Executor, OpType)
Definition: logichelpers.h:223
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > downloadMasterIndex(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, zypp::filesystem::Pathname masterIndex_r)
AsyncOpRef< expected< zypp::RepoStatus > > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
Definition: plaindir.cc:42
DlContextRefType _dlContext
typename conditional< B, T, F >::type conditional_t
Definition: TypeTraits.h:39
std::conditional_t< isAsync, AsyncOpRef< T >, T > makeReadyResult(T &&result)
Definition: asyncop.h:297
static expected success(ConsParams &&...params)
Definition: expected.h:115
std::shared_ptr< AsyncOp< T > > AsyncOpRef
Definition: asyncop.h:255
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
Definition: PublicKey.h:364
Reads through a repomd.xml file and collects type, location, checksum and other data about metadata f...
AsyncOpRef< expected< zypp::RepoStatus > > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
Base class for Exception.
Definition: Exception.h:146
expected< zypp::keyring::VerifyFileContext > verifySignature(SyncContextRef ctx, zypp::keyring::VerifyFileContext context)
static bool isSafeKeyId(const std::string &id_r)
!<
Definition: PublicKey.h:432
AsyncOpRef< expected< zypp::RepoStatus > > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
Definition: rpmmd.cc:74
AsyncOpRef< expected< zypp::RepoStatus > > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
Definition: susetags.cc:84
auto and_then(Fun &&function)
Definition: expected.h:623
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:221
Interface of repomd.xml file reader.
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
Definition: rpmmd.cc:171
ResultType and_then(const expected< T, E > &exp, Function &&f)
Definition: expected.h:423
Track changing files or directories.
Definition: RepoStatus.h:40
static PublicKey noThrow(const Pathname &keyFile_r)
Static ctor returning an empty PublicKey rather than throwing.
Definition: PublicKey.cc:640
#define ZYPP_FWD_CURRENT_EXCPT()
Drops a logline and returns the current Exception as a std::exception_ptr.
Definition: Exception.h:436
#define DBG
Definition: Logger.h:99
zypp::Pathname _destdir
std::vector< zypp::PublicKeyData > _buddyKeys