libzypp: zypp/ng/repo/workflows/susetags.cc Source File

libzypp 17.36.6

susetags.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
9#include "susetags.h"
10#include "zypp-core/base/Regex.h"
11#include <zypp-core/zyppng/ui/ProgressObserver>
12#include <zypp-media/ng/ProvideSpec>
13#include <zypp/ng/Context>
14
15#include <zypp-core/parser/ParseException>
16
17#include <zypp/parser/susetags/ContentFileReader.h>
18#include <zypp/parser/susetags/RepoIndex.h>
19
20#include <zypp/ng/workflows/logichelpers.h>
21#include <zypp/ng/workflows/contextfacade.h>
22#include <zypp/ng/repo/workflows/repodownloaderwf.h>
23#include <zypp/ng/workflows/checksumwf.h>
24
25#undef ZYPP_BASE_LOGGER_LOGGROUP
26#define ZYPP_BASE_LOGGER_LOGGROUP "zypp::repomanager"
27
28 namespace zyppng::SuseTagsWorkflows {
29
30 namespace {
31
32 using namespace zyppng::operators;
33
39 template<class Executor, class OpType>
40 struct StatusLogic : public LogicBase<Executor, OpType>{
41 ZYPP_ENABLE_LOGIC_BASE(Executor, OpType);
42
43 public:
44
45 using DlContextRefType = std::conditional_t<zyppng::detail::is_async_op_v<OpType>, repo::AsyncDownloadContextRef, repo::SyncDownloadContextRef>;
46 using ZyppContextType = typename DlContextRefType::element_type::ContextType;
47 using ProvideType = typename ZyppContextType::ProvideType;
48 using MediaHandle = typename ProvideType::MediaHandle;
49 using ProvideRes = typename ProvideType::Res;
50
51 StatusLogic( DlContextRefType ctx, MediaHandle &&media )
52 : _ctx(std::move(ctx))
53 , _handle(std::move(media))
54 {}
55
56 MaybeAsyncRef<expected<zypp::RepoStatus>> execute() {
57 return _ctx->zyppContext()->provider()->provide( _handle, _ctx->repoInfo().path() / "content" , ProvideFileSpec() )
58 | [this]( expected<ProvideRes> contentFile ) {
59
60 // mandatory master index is missing -> stay empty
61 if ( !contentFile )
62 return makeReadyResult( make_expected_success (zypp::RepoStatus() ));
63
64 zypp::RepoStatus status ( contentFile->file() );
65
66 if ( !status.empty() /* && _ctx->repoInfo().requireStatusWithMediaFile() */ ) {
67 return _ctx->zyppContext()->provider()->provide( _handle, "/media.1/media" , ProvideFileSpec())
68 | [status = std::move(status)]( expected<ProvideRes> mediaFile ) mutable {
69 if ( mediaFile ) {
70 return make_expected_success(status && zypp::RepoStatus( mediaFile->file()) );
71 }
72 return make_expected_success( std::move(status) );
73 };
74 }
75 return makeReadyResult( make_expected_success(std::move(status)) );
76 };
77 }
78
79 DlContextRefType _ctx;
80 MediaHandle _handle;
81 };
82 }
83
84 AsyncOpRef<expected<zypp::RepoStatus> > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
85 {
86 return SimpleExecutor< StatusLogic, AsyncOp<expected<zypp::RepoStatus>> >::run( std::move(dl), std::move(mediaHandle) );
87 }
88
89 expected<zypp::RepoStatus> repoStatus(repo::SyncDownloadContextRef dl, SyncMediaHandle mediaHandle)
90 {
91 return SimpleExecutor< StatusLogic, SyncOp<expected<zypp::RepoStatus>> >::run( std::move(dl), std::move(mediaHandle) );
92 }
93
94
95
96 namespace {
97
98 using namespace zyppng::operators;
99
100 // search old repository file file to run the delta algorithm on
101 static zypp::Pathname search_deltafile( const zypp::Pathname &dir, const zypp::Pathname &file )
102 {
103 zypp::Pathname deltafile(dir + file.basename());
104 if (zypp::PathInfo(deltafile).isExist())
105 return deltafile;
106 return zypp::Pathname();
107 }
108
109
110 template<class Executor, class OpType>
111 struct DlLogic : public LogicBase<Executor, OpType> {
112
113 ZYPP_ENABLE_LOGIC_BASE(Executor, OpType);
114 public:
115
116 using DlContextRefType = std::conditional_t<zyppng::detail::is_async_op_v<OpType>, repo::AsyncDownloadContextRef, repo::SyncDownloadContextRef>;
117 using ZyppContextType = typename DlContextRefType::element_type::ContextType;
118 using ProvideType = typename ZyppContextType::ProvideType;
119 using MediaHandle = typename ProvideType::MediaHandle;
120 using ProvideRes = typename ProvideType::Res;
121
122 DlLogic( DlContextRefType ctx, MediaHandle &&mediaHandle, ProgressObserverRef &&progressObserver )
123 : _ctx( std::move(ctx))
124 , _mediaHandle(std::move(mediaHandle))
125 , _progressObserver(std::move(progressObserver))
126 {}
127
128 auto execute() {
129 // download media file here
130 return RepoDownloaderWorkflow::downloadMediaInfo( _mediaHandle, _ctx->destDir() )
131 | [this]( expected<zypp::ManagedFile> &&mediaInfo ) {
132
133 // remember the media info if we had one
134 if ( mediaInfo ) _ctx->files().push_back ( std::move(mediaInfo.get()) );
135
136 if ( _progressObserver ) _progressObserver->inc();
137
138 return RepoDownloaderWorkflow::downloadMasterIndex( _ctx, _mediaHandle, _ctx->repoInfo().path() / "content" )
139 | inspect( incProgress( _progressObserver ) )
140 | and_then( [this] ( DlContextRefType && ) {
141
142 zypp::Pathname contentPath = _ctx->files().front();
143 std::vector<zypp::OnMediaLocation> requiredFiles;
144
145 // Content file first to get the repoindex
146 try {
147 const zypp::Pathname &inputfile = contentPath;
148 zypp::parser::susetags::ContentFileReader content;
149 content.setRepoIndexConsumer( [this]( const auto & data_r ) {
150 MIL << "Consuming repo index" << std::endl;
151 _repoindex = data_r;
152 });
153 content.parse( inputfile );
154
155 if ( ! _repoindex ) {
156 ZYPP_THROW( zypp::parser::ParseException( ( _ctx->destDir() / _ctx->repoInfo().path() ).asString() + ": " + "No repository index in content file." ) );
157 }
158
159 MIL << "RepoIndex: " << _repoindex << std::endl;
160 if ( _repoindex->metaFileChecksums.empty() ) {
161 ZYPP_THROW( zypp::parser::ParseException( ( _ctx->destDir() / _ctx->repoInfo().path() ).asString() + ": " + "No metadata checksums in content file." ) );
162 }
163
164 if ( _repoindex->signingKeys.empty() ) {
165 WAR << "No signing keys defined." << std::endl;
166 }
167
168 // Prepare parsing
169 zypp::Pathname descr_dir = _repoindex->descrdir; // path below reporoot
170 //_datadir = _repoIndex->datadir; // path below reporoot
171
172 std::map<std::string,zypp::parser::susetags::RepoIndex::FileChecksumMap::const_iterator> availablePackageTranslations;
173
174 for_( it, _repoindex->metaFileChecksums.begin(), _repoindex->metaFileChecksums.end() )
175 {
176 // omit unwanted translations
177 if ( zypp::str::hasPrefix( it->first, "packages" ) )
178 {
179 static const zypp::str::regex rx_packages( "^packages((.gz)?|(.([^.]*))(.gz)?)$" );
180 zypp::str::smatch what;
181
182 if ( zypp::str::regex_match( it->first, what, rx_packages ) ) {
183 if ( what[4].empty() // packages(.gz)?
184 || what[4] == "DU"
185 || what[4] == "en" )
186 { ; /* always downloaded */ }
187 else if ( what[4] == "FL" )
188 { continue; /* never downloaded */ }
189 else
190 {
191 // remember and decide later
192 availablePackageTranslations[what[4]] = it;
193 continue;
194 }
195 }
196 else
197 continue; // discard
198
199 } else if ( it->first == "patterns.pat"
200 || it->first == "patterns.pat.gz" ) {
201 // take all patterns in one go
202
203 } else if ( zypp::str::endsWith( it->first, ".pat" )
204 || zypp::str::endsWith( it->first, ".pat.gz" ) ) {
205
206 // *** see also zypp/parser/susetags/RepoParser.cc ***
207
208 // omit unwanted patterns, see https://bugzilla.novell.com/show_bug.cgi?id=298716
209 // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
210 // split at dots, take .pat or .pat.gz into account
211
212 std::vector<std::string> patparts;
213 unsigned archpos = 2;
214 // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
215 unsigned count = zypp::str::split( it->first, std::back_inserter(patparts), "." );
216 if ( patparts[count-1] == "gz" )
217 archpos++;
218
219 if ( count > archpos ) {
220 try { // might by an invalid architecture
221 zypp::Arch patarch( patparts[count-archpos] );
222 if ( !patarch.compatibleWith( zConfig().systemArchitecture() ) ) {
223 // discard, if not compatible
224 MIL << "Discarding pattern " << it->first << std::endl;
225 continue;
226 }
227
228 } catch ( const zypp::Exception & excpt ) {
229 WAR << "Pattern file name does not contain recognizable architecture: " << it->first << std::endl;
230 // keep .pat file if it doesn't contain an recognizable arch
231 }
232 }
233 }
234
235 MIL << "adding job " << it->first << std::endl;
236 auto location = zypp::OnMediaLocation( repoInfo().path() + descr_dir + it->first, 1 )
237 .setChecksum( it->second )
238 .setDeltafile( search_deltafile( _ctx->deltaDir() + descr_dir, it->first) );
239
240 requiredFiles.push_back( std::move(location) );
241 }
242
243
244 // check whether to download more package translations:
245 {
246 auto fnc_checkTransaltions( [&]( const zypp::Locale & locale_r ) {
247 for ( zypp::Locale toGet( locale_r ); toGet; toGet = toGet.fallback() ) {
248 auto it( availablePackageTranslations.find( toGet.code() ) );
249 if ( it != availablePackageTranslations.end() ) {
250 auto mit( it->second );
251 MIL << "adding job " << mit->first << std::endl;
252 requiredFiles.push_back( zypp::OnMediaLocation( repoInfo().path() + descr_dir + mit->first, 1 )
253 .setChecksum( mit->second )
254 .setDeltafile( search_deltafile( deltaDir() + descr_dir, mit->first) ));
255 break;
256 }
257 }
258 });
259
260 for ( const zypp::Locale & it : zConfig().repoRefreshLocales() ) {
261 fnc_checkTransaltions( it );
262 }
263 fnc_checkTransaltions( zConfig().textLocale() );
264 }
265
266 for( const auto &it : _repoindex->mediaFileChecksums ) {
267 // Repo adopts license files listed in HASH
268 if ( it.first != "license.tar.gz" )
269 continue;
270
271 MIL << "adding job " << it.first << std::endl;
272 requiredFiles.push_back( zypp::OnMediaLocation ( repoInfo().path() + it.first, 1 )
273 .setChecksum( it.second )
274 .setDeltafile( search_deltafile( deltaDir(), it.first ) ));
275 }
276
277 for( const auto &it : _repoindex->signingKeys ) {
278 MIL << "adding job " << it.first << std::endl;
279 zypp::OnMediaLocation location( repoInfo().path() + it.first, 1 );
280 location.setChecksum( it.second );
281 requiredFiles.push_back( std::move(location) );
282 }
283
284 } catch ( const zypp::Exception &e ) {
285 ZYPP_CAUGHT( e );
286 return makeReadyResult(expected<DlContextRefType>::error( ZYPP_FWD_CURRENT_EXCPT() ) );
287 } catch ( ... ) {
288 return makeReadyResult(expected<DlContextRefType>::error( ZYPP_FWD_CURRENT_EXCPT() ) );
289 }
290
291 // add the required files to the base steps
292 if ( _progressObserver ) _progressObserver->setBaseSteps ( _progressObserver->baseSteps () + requiredFiles.size() );
293
294 return transform_collect ( std::move(requiredFiles), [this]( zypp::OnMediaLocation file ) {
295
296 return DownloadWorkflow::provideToCacheDir( _ctx, _mediaHandle, file.filename(), ProvideFileSpec(file) )
297 | inspect ( incProgress( _progressObserver ) );
298
299 }) | and_then ( [this]( std::vector<zypp::ManagedFile> &&dlFiles ) {
300 auto &downloadedFiles = _ctx->files();
301 downloadedFiles.insert( downloadedFiles.end(), std::make_move_iterator(dlFiles.begin()), std::make_move_iterator(dlFiles.end()) );
302 return expected<DlContextRefType>::success( std::move(_ctx) );
303 });
304 });
305
306 } | finishProgress( _progressObserver );
307 }
308
309 private:
310
311 const zypp::RepoInfo &repoInfo() const {
312 return _ctx->repoInfo();
313 }
314
315 const zypp::filesystem::Pathname &deltaDir() const {
316 return _ctx->deltaDir();
317 }
318
319 zypp::ZConfig &zConfig() {
320 return _ctx->zyppContext()->config();
321 }
322
323 DlContextRefType _ctx;
324 zypp::parser::susetags::RepoIndex_Ptr _repoindex;
325 MediaHandle _mediaHandle;
326 ProgressObserverRef _progressObserver;
327 };
328 }
329
330 AsyncOpRef<expected<repo::AsyncDownloadContextRef> > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
331 {
332 return SimpleExecutor< DlLogic, AsyncOp<expected<repo::AsyncDownloadContextRef>> >::run( std::move(dl), std::move(mediaHandle), std::move(progressObserver) );
333 }
334
335 expected<repo::SyncDownloadContextRef> download(repo::SyncDownloadContextRef dl, SyncMediaHandle mediaHandle, ProgressObserverRef progressObserver)
336 {
337 return SimpleExecutor< DlLogic, SyncOp<expected<repo::SyncDownloadContextRef>> >::run( std::move(dl), std::move(mediaHandle), std::move(progressObserver) );
338 }
339
340}
ContentFileReader.h
RepoIndex.h
checksumwf.h
zypp::OnMediaLocation::filename
const Pathname & filename() const
The path to the resource on the medium.
Definition onmedialocation.cc:88
zypp::RepoStatus
Track changing files or directories.
Definition RepoStatus.h:41
zypp::RepoStatus::empty
bool empty() const
Whether the status is empty (empty checksum)
Definition RepoStatus.cc:234
zypp::filesystem::PathInfo
Wrapper class for stat/lstat.
Definition PathInfo.h:226
zypp::filesystem::Pathname
Pathname.
Definition Pathname.h:47
zypp::filesystem::Pathname::basename
std::string basename() const
Return the last component of this path.
Definition Pathname.h:130
zypp::parser::susetags::ContentFileReader::parse
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
Definition ContentFileReader.cc:175
zypp::parser::susetags::ContentFileReader::setRepoIndexConsumer
void setRepoIndexConsumer(const RepoIndexConsumer &fnc_r)
Consumer to call when repo index was parsed.
Definition ContentFileReader.h:66
zyppng::ProvideFileSpec
Definition providespec.h:92
zyppng::ProvideMediaHandle
Definition provide.h:52
zyppng::ProvideRes
A ProvideRes object is a reference counted ownership of a resource in the cache provided by a Provide...
Definition provideres.h:36
zyppng::SyncMediaHandle
Definition mediafacade.h:26
zyppng::expected
Definition expected.h:27
zyppng::expected::success
static expected success(ConsParams &&...params)
Definition expected.h:115
zyppng::expected::error
E & error()
Definition expected.h:225
contextfacade.h
logichelpers.h
ZYPP_ENABLE_LOGIC_BASE
#define ZYPP_ENABLE_LOGIC_BASE(Executor, OpType)
Definition logichelpers.h:223
std
Definition Arch.h:364
std::conditional_t
typename conditional< B, T, F >::type conditional_t
Definition TypeTraits.h:39
zypp::media
Definition MediaCD.cc:47
zypp::sat::empty
bool empty() const
Whether neither idents nor provides are set.
zypp::str::hasPrefix
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition String.h:1026
zypp::str::endsWith
bool endsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasSuffix
Definition String.h:1091
zypp::str::regex_match
bool regex_match(const std::string &s, smatch &matches, const regex &regex)
\relates regex \ingroup ZYPP_STR_REGEX \relates regex \ingroup ZYPP_STR_REGEX
Definition Regex.h:70
zypp::str::split
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \t", const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition String.h:531
zyppng::DownloadWorkflow::provideToCacheDir
AsyncOpRef< expected< zypp::ManagedFile > > provideToCacheDir(AsyncCacheProviderContextRef cacheContext, ProvideMediaHandle medium, zypp::Pathname file, ProvideFileSpec filespec)
Definition downloadwf.cc:209
zyppng::RepoDownloaderWorkflow::downloadMasterIndex
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > downloadMasterIndex(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, zypp::filesystem::Pathname masterIndex_r)
Definition repodownloaderwf.cc:294
zyppng::RepoDownloaderWorkflow::downloadMediaInfo
auto downloadMediaInfo(MediaHandle &&mediaHandle, const zypp::filesystem::Pathname &destdir)
Definition repodownloaderwf.h:53
zyppng::SuseTagsWorkflows
Download workflow namespace for SUSETags (YaST2) repositories Encapsulates all the knowledge of which...
Definition susetags.cc:28
zyppng::SuseTagsWorkflows::download
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
Definition susetags.cc:330
zyppng::SuseTagsWorkflows::repoStatus
AsyncOpRef< expected< zypp::RepoStatus > > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
Definition susetags.cc:84
zyppng::operators
Definition asyncresult.h:279
zyppng::operators::transform_collect
auto transform_collect(Transformation &&f)
Definition expected.h:706
zyppng::operators::and_then
auto and_then(Fun &&function)
Definition expected.h:623
zyppng::operators::finishProgress
auto finishProgress(ProgressObserverRef progressObserver, ProgressObserver::FinishResult result=ProgressObserver::Success)
Definition progressobserver.h:252
zyppng::operators::incProgress
auto incProgress(ProgressObserverRef progressObserver, double progrIncrease=1.0, std::optional< std::string > newStr={})
Definition progressobserver.h:229
zyppng::operators::inspect
auto inspect(Fun &&function)
Definition expected.h:637
zyppng::makeReadyResult
std::conditional_t< isAsync, AsyncOpRef< T >, T > makeReadyResult(T &&result)
Definition asyncop.h:297
zyppng::AsyncOpRef
std::shared_ptr< AsyncOp< T > > AsyncOpRef
Definition asyncop.h:255
zyppng::make_expected_success
static expected< std::decay_t< Type >, Err > make_expected_success(Type &&t)
Definition expected.h:397
repodownloaderwf.h
zyppng::LogicBase
Definition logichelpers.h:161
zyppng::SimpleExecutor
Definition logichelpers.h:212
susetags.h
for_
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition Easy.h:27
ZYPP_CAUGHT
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition Exception.h:440
ZYPP_FWD_CURRENT_EXCPT
#define ZYPP_FWD_CURRENT_EXCPT()
Drops a logline and returns the current Exception as a std::exception_ptr.
Definition Exception.h:436
ZYPP_THROW
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition Exception.h:424
MIL
#define MIL
Definition Logger.h:100
WAR
#define WAR
Definition Logger.h:101
Regex.h