apt: apt-pkg/acquire.h Source File

apt 2.7.5 commandline package manager

acquire.h
Go to the documentation of this file.
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
3/* ######################################################################
4
5 Acquire - File Acquiration
6
7 This module contains the Acquire system. It is responsible for bringing
8 files into the local pathname space. It deals with URIs for files and
9 URI handlers responsible for downloading or finding the URIs.
10
11 Each file to download is represented by an Acquire::Item class subclassed
12 into a specialization. The Item class can add itself to several URI
13 acquire queues each prioritized by the download scheduler. When the
14 system is run the proper URI handlers are spawned and the acquire
15 queues are fed into the handlers by the schedular until the queues are
16 empty. This allows for an Item to be downloaded from an alternate source
17 if the first try turns out to fail. It also allows concurrent downloading
18 of multiple items from multiple sources as well as dynamic balancing
19 of load between the sources.
20
21 Scheduling of downloads is done on a first ask first get basis. This
22 preserves the order of the download as much as possible. And means the
23 fastest source will tend to process the largest number of files.
24
25 Internal methods and queues for performing gzip decompression,
26 md5sum hashing and file copying are provided to allow items to apply
27 a number of transformations to the data files they are working with.
28
29 ##################################################################### */
30 /*}}}*/
31
/*}}}*/
57
65#ifndef PKGLIB_ACQUIRE_H
66#define PKGLIB_ACQUIRE_H
67
68#include <apt-pkg/hashes.h>
69#include <apt-pkg/macros.h>
70#include <apt-pkg/weakptr.h>
71
72#include <chrono>
73#include <string>
74#include <vector>
75
76#include <stddef.h>
77#include <sys/select.h>
78#include <sys/time.h>
79
80
81
82class pkgAcquireStatus;
83class metaIndex;
84
93class APT_PUBLIC pkgAcquire
94{
95 private:
97 using clock = std::chrono::steady_clock;
99 using time_point = std::chrono::time_point<clock>;
101 int LockFD;
103 void * const d;
104
105 public:
106
107 class Item;
108 class Queue;
109 class Worker;
110 struct MethodConfig;
111 struct ItemDesc;
112 friend class Item;
113 friend class pkgAcqMetaBase;
114 friend class Queue;
115
116 typedef std::vector<Item *>::iterator ItemIterator;
117 typedef std::vector<Item *>::const_iterator ItemCIterator;
118
119 protected:
120
126 std::vector<Item *> Items;
127
133 Queue *Queues;
134
140 Worker *Workers;
141
152 MethodConfig *Configs;
153
155 pkgAcquireStatus *Log;
156
158 unsigned long ToFetch;
159
160 // Configurable parameters for the scheduler
161
163 enum QueueStrategy {
167 QueueHost,
171 QueueAccess} QueueMode;
172
174 bool const Debug;
176 bool Running;
177
179 void Add(Item *Item);
180
182 void Remove(Item *Item);
183
185 void Add(Worker *Work);
186
188 void Remove(Worker *Work);
189
196 void Enqueue(ItemDesc &Item);
197
199 void Dequeue(Item *Item);
200
211 std::string QueueName(std::string URI,MethodConfig const *&Config);
212
227 virtual void SetFds(int &Fd,fd_set *RSet,fd_set *WSet);
228
241 virtual bool RunFds(fd_set *RSet,fd_set *WSet);
242
249 void Bump();
250
251 public:
252
259 MethodConfig *GetConfig(std::string Access);
260
262 enum RunResult {
264 Continue,
265
267 Failed,
268
272 Cancelled};
273
285 RunResult Run(int PulseInterval=500000);
286
290 void Shutdown();
291
296 inline Worker *WorkersBegin() {return Workers;};
297
303 Worker *WorkerStep(Worker *I) APT_PURE;
304
306 inline ItemIterator ItemsBegin() {return Items.begin();};
307 inline ItemCIterator ItemsBegin() const {return Items.begin();};
308
310 inline ItemIterator ItemsEnd() {return Items.end();};
311 inline ItemCIterator ItemsEnd() const {return Items.end();};
312
313 // Iterate over queued Item URIs
314 class UriIterator;
320 UriIterator UriBegin();
322 UriIterator UriEnd();
323
332 bool Clean(std::string Dir);
333
337 unsigned long long TotalNeeded();
338
342 unsigned long long FetchNeeded();
343
347 unsigned long long PartialPresent();
348
349 void SetLog(pkgAcquireStatus *Progress) { Log = Progress; }
350
358 bool GetLock(std::string const &Lock);
359
361 explicit pkgAcquire(pkgAcquireStatus *Log);
362 pkgAcquire();
363
369 virtual ~pkgAcquire();
370
371 APT_HIDDEN static std::string URIEncode(std::string const &part);
372
373 private:
374 APT_HIDDEN void Initialize();
375};
376
382struct APT_PUBLIC pkgAcquire::ItemDesc : public WeakPointable
383{
385 std::string URI;
387 std::string Description;
389 std::string ShortDesc;
391 Item *Owner;
392};
393 /*}}}*/
398class APT_PUBLIC pkgAcquire::Queue
399{
400 friend class pkgAcquire;
401 friend class pkgAcquire::UriIterator;
402 friend class pkgAcquire::Worker;
403
405 void * const d;
406
408 Queue *Next;
409
410 protected:
411
413 struct QItem : public ItemDesc
414 {
416 QItem *Next;
418 pkgAcquire::Worker *Worker;
419
421 std::vector<Item*> Owners;
422
426 unsigned long long CurrentSize = 0;
427
431 unsigned long long TotalSize = 0;
432
436 unsigned long long ResumePoint = 0;
437
438 typedef std::vector<Item*>::const_iterator owner_iterator;
439
443 void operator =(pkgAcquire::ItemDesc const &I)
444 {
445 URI = I.URI;
446 Description = I.Description;
447 ShortDesc = I.ShortDesc;
448 Owners.clear();
449 Owners.push_back(I.Owner);
450 Owner = I.Owner;
451 };
452
454 HashStringList GetExpectedHashes() const;
455
457 unsigned long long GetMaximumSize() const;
458
460 void SyncDestinationFiles() const;
461
463 std::string Custom600Headers() const;
465 int APT_HIDDEN GetPriority() const;
467 time_point APT_HIDDEN GetFetchAfter() const;
468 };
469
471 std::string Name;
472
477 QItem *Items;
478
488 pkgAcquire::Worker *Workers;
489
491 pkgAcquire *Owner;
492
496 signed long PipeDepth;
497
501 unsigned long MaxPipeDepth;
502
503 public:
504
510 bool Enqueue(ItemDesc &Item);
511
516 bool Dequeue(Item *Owner);
517
526 QItem *FindItem(std::string URI,pkgAcquire::Worker *Owner) APT_PURE;
527
532 bool ItemStart(QItem *Itm,unsigned long long Size);
533
544 bool ItemDone(QItem *Itm);
545
553 bool Startup();
554
564 bool Shutdown(bool Final);
565
570 bool Cycle();
571
582 void Bump();
583
589 Queue(std::string const &Name,pkgAcquire * const Owner);
590
594 virtual ~Queue();
595};
596 /*}}}*/
598class APT_PUBLIC pkgAcquire::UriIterator
599{
601 void * const d;
602
604 pkgAcquire::Queue *CurQ;
606 pkgAcquire::Queue::QItem *CurItem;
607
608 public:
609
610 inline void operator ++() {operator ++(0);};
611
612 void operator ++(int)
613 {
614 CurItem = CurItem->Next;
615 while (CurItem == 0 && CurQ != 0)
616 {
617 CurItem = CurQ->Items;
618 CurQ = CurQ->Next;
619 }
620 };
621
622 inline pkgAcquire::Queue::QItem const *operator ->() const {return CurItem;};
623 inline bool operator !=(UriIterator const &rhs) const {return rhs.CurQ != CurQ || rhs.CurItem != CurItem;};
624 inline bool operator ==(UriIterator const &rhs) const {return rhs.CurQ == CurQ && rhs.CurItem == CurItem;};
625
630 explicit UriIterator(pkgAcquire::Queue *Q);
631 virtual ~UriIterator();
632};
633 /*}}}*/
635struct APT_PUBLIC pkgAcquire::MethodConfig
636{
637 class Private;
639 Private *const d;
640
645 MethodConfig *Next;
646
648 std::string Access;
649
651 std::string Version;
652
656 bool SingleInstance;
657
659 bool Pipeline;
660
665 bool SendConfig;
666
670 bool LocalOnly;
671
678 bool NeedsCleanup;
679
681 bool Removable;
682
688 MethodConfig();
689
690 APT_HIDDEN bool GetAuxRequests() const;
691 APT_HIDDEN void SetAuxRequests(bool const value);
692 APT_HIDDEN bool GetSendURIEncoded() const;
693 APT_HIDDEN void SetSendURIEncoded(bool const value);
694
695 virtual ~MethodConfig();
696};
697 /*}}}*/
702 class APT_PUBLIC pkgAcquireStatus
703{
705 void * const d;
706
707 protected:
708
710 struct timeval Time;
711
713 struct timeval StartTime;
714
718 unsigned long long LastBytes;
719
723 unsigned long long CurrentCPS;
724
728 unsigned long long CurrentBytes;
729
735 unsigned long long TotalBytes;
736
740 unsigned long long FetchedBytes;
741
745 unsigned long long ElapsedTime;
746
752 unsigned long TotalItems;
753
755 unsigned long CurrentItems;
756
759 double Percent;
760
761 public:
762
766 bool Update;
767
774 bool MorePulses;
775
782 virtual void Fetched(unsigned long long Size,unsigned long long ResumePoint);
783
801 virtual bool MediaChange(std::string Media,std::string Drive) = 0;
802
803 struct ReleaseInfoChange
804 {
805 std::string Type;
806 std::string From;
807 std::string To;
808 std::string Message;
809 bool DefaultAction;
810 };
828 virtual bool ReleaseInfoChanges(metaIndex const * const LastRelease, metaIndex const * const CurrentRelease, std::vector<ReleaseInfoChange> &&Changes);
829 APT_HIDDEN static bool ReleaseInfoChangesAsGlobalErrors(std::vector<ReleaseInfoChange> &&Changes);
830
836 virtual void IMSHit(pkgAcquire::ItemDesc &/*Itm*/) {};
837
839 virtual void Fetch(pkgAcquire::ItemDesc &/*Itm*/) {};
840
842 virtual void Done(pkgAcquire::ItemDesc &/*Itm*/) {};
843
847 virtual void Fail(pkgAcquire::ItemDesc &/*Itm*/) {};
848
859 virtual bool Pulse(pkgAcquire *Owner);
860
862 virtual void Start();
863
865 virtual void Stop();
866
868 pkgAcquireStatus();
869 virtual ~pkgAcquireStatus();
870};
871 /*}}}*/
874#endif
HashStringList
Definition hashes.h:79
URI
Definition strutl.h:212
WeakPointable
Definition weakptr.h:34
metaIndex
Definition metaindex.h:21
pkgAcqMetaBase
the manager of a transaction
Definition acquire-item.h:408
pkgAcquireStatus
A monitor object for downloads controlled by the pkgAcquire class. {{{.
Definition acquire.h:703
pkgAcquireStatus::IMSHit
virtual void IMSHit(pkgAcquire::ItemDesc &)
Invoked when an item is confirmed to be up-to-date.
Definition acquire.h:836
pkgAcquireStatus::ElapsedTime
unsigned long long ElapsedTime
The amount of time that has elapsed since the download started.
Definition acquire.h:745
pkgAcquireStatus::Percent
double Percent
The estimated percentage of the download (0-100)
Definition acquire.h:759
pkgAcquireStatus::Fail
virtual void Fail(pkgAcquire::ItemDesc &)
Invoked when the process of fetching an item encounters a fatal error.
Definition acquire.h:847
pkgAcquireStatus::Done
virtual void Done(pkgAcquire::ItemDesc &)
Invoked when an item is successfully and completely fetched.
Definition acquire.h:842
pkgAcquireStatus::MediaChange
virtual bool MediaChange(std::string Media, std::string Drive)=0
Invoked when the user should be prompted to change the inserted removable media.
pkgAcquireStatus::Fetch
virtual void Fetch(pkgAcquire::ItemDesc &)
Invoked when some of an item's data is fetched.
Definition acquire.h:839
pkgAcquireStatus::FetchedBytes
unsigned long long FetchedBytes
The total number of bytes accounted for by items that were successfully fetched.
Definition acquire.h:740
pkgAcquireStatus::Update
bool Update
If true, the download scheduler should call Pulse() at the next available opportunity.
Definition acquire.h:766
pkgAcquireStatus::CurrentBytes
unsigned long long CurrentBytes
The number of bytes fetched as of the most recent call to pkgAcquireStatus::Pulse,...
Definition acquire.h:728
pkgAcquireStatus::MorePulses
bool MorePulses
If true, extra Pulse() invocations will be performed.
Definition acquire.h:774
pkgAcquireStatus::CurrentItems
unsigned long CurrentItems
The number of items that have been successfully downloaded.
Definition acquire.h:755
pkgAcquireStatus::TotalBytes
unsigned long long TotalBytes
The total number of bytes that need to be fetched.
Definition acquire.h:735
pkgAcquireStatus::CurrentCPS
unsigned long long CurrentCPS
The current rate of download as of the most recent call to pkgAcquireStatus::Pulse,...
Definition acquire.h:723
pkgAcquireStatus::LastBytes
unsigned long long LastBytes
The number of bytes fetched as of the previous call to pkgAcquireStatus::Pulse, including local items...
Definition acquire.h:718
pkgAcquireStatus::TotalItems
unsigned long TotalItems
The total number of items that need to be fetched.
Definition acquire.h:752
pkgAcquire
Represents the process by which a pkgAcquire object should retrieve a file or a collection of files.
Definition acquire-item.h:59
pkgAcquire::Shutdown
bool Shutdown(bool Final)
Shut down the worker process associated with this queue.
pkgAcquire::ItemsEnd
ItemIterator ItemsEnd()
Get the end iterator of the list of items.
Definition acquire.h:310
pkgAcquire::Pipeline
bool Pipeline
If true, this method supports pipelined downloading.
Definition acquire.h:659
pkgAcquire::NeedsCleanup
bool NeedsCleanup
If true, the subprocess has to carry out some cleanup actions before shutting down.
Definition acquire.h:678
pkgAcquire::Debug
bool const Debug
If true, debugging information will be dumped to std::clog.
Definition acquire.h:174
pkgAcquire::MethodConfig
MethodConfig()
Set up the default method parameters.
pkgAcquire::Enqueue
bool Enqueue(ItemDesc &Item)
Insert the given fetch request into this queue.
pkgAcquire::Items
QItem * Items
The head of the list of items contained in this queue.
Definition acquire.h:477
pkgAcquire::Cycle
bool Cycle()
Send idle items to the worker process.
pkgAcquire::WorkersBegin
Worker * WorkersBegin()
Get the first Worker object.
Definition acquire.h:296
pkgAcquire::ItemDone
bool ItemDone(QItem *Itm)
Remove the given item from this queue and set its state to pkgAcquire::Item::StatDone.
pkgAcquire::Description
std::string Description
description of this item.
Definition acquire.h:387
pkgAcquire::SingleInstance
bool SingleInstance
If true, only one download queue should be created for this method.
Definition acquire.h:656
pkgAcquire::FindItem
QItem * FindItem(std::string URI, pkgAcquire::Worker *Owner) APT_PURE
Locate an item in this queue.
pkgAcquire::ItemStart
bool ItemStart(QItem *Itm, unsigned long long Size)
pkgAcquire::Owner
pkgAcquire * Owner
the download scheduler with which this queue is associated.
Definition acquire.h:491
pkgAcquire::Items
std::vector< Item * > Items
A list of items to download.
Definition acquire.h:126
pkgAcquire::Removable
bool Removable
If true, this fetch method acquires files from removable media.
Definition acquire.h:681
pkgAcquire::Dequeue
bool Dequeue(Item *Owner)
Remove all fetch requests for the given item from this queue.
pkgAcquire::LocalOnly
bool LocalOnly
If true, this fetch method does not require network access; all files are to be acquired from the loc...
Definition acquire.h:670
pkgAcquire::Version
std::string Version
The implementation version of this acquire method.
Definition acquire.h:651
pkgAcquire::PipeDepth
signed long PipeDepth
The number of entries in this queue that are currently being downloaded.
Definition acquire.h:496
pkgAcquire::ShortDesc
std::string ShortDesc
shorter description of this item.
Definition acquire.h:389
pkgAcquire::Queue
Queue(std::string const &Name, pkgAcquire *const Owner)
Create a new Queue.
pkgAcquire::Owner
Item * Owner
underlying item which is to be downloaded.
Definition acquire.h:391
pkgAcquire::Queues
Queue * Queues
The head of the list of active queues.
Definition acquire.h:133
pkgAcquire::URI
std::string URI
URI from which to download this item.
Definition acquire.h:385
pkgAcquire::Configs
MethodConfig * Configs
The head of the list of acquire method configurations.
Definition acquire.h:152
pkgAcquire::Startup
bool Startup()
Start the worker process associated with this queue.
pkgAcquire::~Queue
virtual ~Queue()
pkgAcquire::ToFetch
unsigned long ToFetch
The number of files which are to be fetched.
Definition acquire.h:158
pkgAcquire::Bump
void Bump()
Check for items that could be enqueued.
pkgAcquire::Worker
Worker(Queue *OwnerQ, MethodConfig *Config, pkgAcquireStatus *Log)
Create a new Worker to download files.
pkgAcquire::RunResult
RunResult
Provides information on how a download terminated.
Definition acquire.h:262
pkgAcquire::Failed
@ Failed
Some files failed to download.
Definition acquire.h:267
pkgAcquire::Continue
@ Continue
All files were fetched successfully.
Definition acquire.h:264
pkgAcquire::UriIterator
UriIterator(pkgAcquire::Queue *Q)
Create a new UriIterator.
pkgAcquire::Name
std::string Name
The name of this queue.
Definition acquire.h:471
pkgAcquire::Next
MethodConfig * Next
The next link on the acquire method list.
Definition acquire.h:645
pkgAcquire::Workers
Worker * Workers
The head of the list of active workers.
Definition acquire.h:140
pkgAcquire::Running
bool Running
If true, a download is currently in progress.
Definition acquire.h:176
pkgAcquire::Workers
pkgAcquire::Worker * Workers
The head of the list of workers associated with this queue.
Definition acquire.h:488
pkgAcquire::SendConfig
bool SendConfig
If true, the worker process should send the entire APT configuration tree to the fetch subprocess whe...
Definition acquire.h:665
pkgAcquire::QueueStrategy
QueueStrategy
Represents the queuing strategy for remote URIs.
Definition acquire.h:163
pkgAcquire::QueueHost
@ QueueHost
Generate one queue for each protocol/host combination; downloads from multiple hosts can proceed in p...
Definition acquire.h:167
pkgAcquire::ItemsBegin
ItemIterator ItemsBegin()
Get the head of the list of items.
Definition acquire.h:306
pkgAcquire::MaxPipeDepth
unsigned long MaxPipeDepth
The maximum number of entries that this queue will attempt to download at once.
Definition acquire.h:501
pkgAcquireStatus::ReleaseInfoChange
Definition acquire.h:804
pkgAcquireStatus::ReleaseInfoChange::From
std::string From
Definition acquire.h:806
pkgAcquireStatus::ReleaseInfoChange::Type
std::string Type
Definition acquire.h:805
pkgAcquireStatus::ReleaseInfoChange::Message
std::string Message
Definition acquire.h:808
pkgAcquireStatus::ReleaseInfoChange::DefaultAction
bool DefaultAction
Definition acquire.h:809
pkgAcquireStatus::ReleaseInfoChange::To
std::string To
Definition acquire.h:807
pkgAcquire::QItem
A single item placed in this queue.
Definition acquire.h:414
pkgAcquire::QItem::GetPriority
int APT_HIDDEN GetPriority() const
pkgAcquire::QItem::GetMaximumSize
unsigned long long GetMaximumSize() const
pkgAcquire::QItem::GetExpectedHashes
HashStringList GetExpectedHashes() const
pkgAcquire::QItem::SyncDestinationFiles
void SyncDestinationFiles() const
get partial files in order
pkgAcquire::QItem::GetFetchAfter
time_point APT_HIDDEN GetFetchAfter() const
pkgAcquire::QItem::Next
QItem * Next
The next item in the queue.
Definition acquire.h:416
pkgAcquire::QItem::Worker
pkgAcquire::Worker * Worker
The worker associated with this item, if any.
Definition acquire.h:418
pkgAcquire::QItem::Owners
std::vector< Item * > Owners
The underlying items interested in the download.
Definition acquire.h:421
pkgAcquire::QItem::Custom600Headers
std::string Custom600Headers() const