File: //lib/python3/dist-packages/awscli/customizations/s3/__pycache__/results.cpython-38.pyc
U
<�^ h � @ s< d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl m
Z
d dl m Z d dl m
Z
d dlmZmZ d d lmZ d d
lmZ d d lmZ d d lmZ d d
lmZ e�e�Zeddddg�Zdefdd�Z e ddg�Z e ddddg�Z e d�Z e ddg�Z e d �Z!ed dg�Z"e d e"d �Z#ed d!d"g�Z$ed#d$g�Z%G d%d&� d&e&�Z'G d'd(� d(e�Z(G d)d*� d*e(�Z)G d+d,� d,e)�Z*G d-d.� d.e(�Z+G d/d0� d0e+�Z,G d1d2� d2e(�Z-G d3d4� d4e(�Z.G d5d6� d6e&�Z/G d7d8� d8e/�Z0G d9d:� d:e/�Z1G d;d<� d<e1�Z2G d=d>� d>e1�Z3G d?d@� d@ej4�Z5G dAdB� dBe&�Z6dS )C� )�divisionN)�
namedtuple)� defaultdict)�CancelledError)�
FatalError)�BaseSubscriber)�queue�ensure_text_type��
relative_path)�human_readable_size)� uni_print)�
WarningResult)�OnDoneFilteredSubscriber�
BaseResult�
transfer_type�src�destc C s, t |j�}|r||7 }t| t| |�|fi �S �N)�list�_fields�typer )�nameZ extra_fields�base_clsZfields� r �B/usr/lib/python3/dist-packages/awscli/customizations/s3/results.py�_create_new_result_cls' s
r � QueuedResult�total_transfer_size�ProgressResult�bytes_transferred� timestamp�
SuccessResult�
FailureResult� exception� DryRunResult� ErrorResult� CtrlCResult)r �
CommandResultZnum_tasks_failedZnum_tasks_warned�FinalTotalSubmissionsResult�total_submissionsc @ s e Zd ZdS )�ShutdownThreadRequestN)�__name__�
__module__� __qualname__r r r r r+ K s r+ c @ sR e Zd ZdZddd�Zdd� Zdd� Zdd � Zd
d � Zd d
� Z dd� Z
dd� Z dS )�BaseResultSubscriberNc C s& || _ i | _|| _|dkr"| j| _dS )z�Subscriber to send result notifications during transfer process
:param result_queue: The queue to place results to be processed later
on.
N)�
_result_queue�_result_kwargs_cache�_transfer_type�
TRANSFER_TYPE)�self� result_queuer r r r �__init__R s
z BaseResultSubscriber.__init__c K s2 | � |� | j|jj }tf |�}| j�|� d S r )�_add_to_result_kwargs_cacher1 �meta� transfer_idr r0 �put)r4 �future�kwargs�
result_kwargsZ
queued_resultr r r � on_queued^ s
z BaseResultSubscriber.on_queuedc K s6 | j |jj }tf |t�� d�|��}| j�|� d S )N)r r! )r1 r8 r9 r �timer0 r: )r4 r; r r< r= Zprogress_resultr r r � on_progressd s ��z BaseResultSubscriber.on_progressc C s | � |�}| j�tf |�� d S r )�%_on_done_pop_from_result_kwargs_cacher0 r: r" �r4 r; r= r r r � _on_successk s
z BaseResultSubscriber._on_successc C sX | � |�}t|t�r:t}t|t�r&t}| j�||d�� n| j�tf d|i|��� d S )N�r$ r$ ) rA �
isinstancer r' r r& r0 r: r# )r4 r; �er= Zerror_result_clsr r r � _on_failureo s
z BaseResultSubscriber._on_failurec C s4 | � |�\}}| j|||jjd�}|| j|jj< d S )N)r r r r )�
_get_src_destr2 r8 �sizer1 r9 )r4 r; r r r= r r r r7 y s �z0BaseResultSubscriber._add_to_result_kwargs_cachec C s | j �|jj�}|�d� |S )Nr )r1 �popr8 r9 rB r r r rA � s
z:BaseResultSubscriber._on_done_pop_from_result_kwargs_cachec C s t d��d S )Nz_get_src_dest()��NotImplementedError)r4 r; r r r rH � s z"BaseResultSubscriber._get_src_dest)N) r, r- r. r3 r6 r> r@ rC rG r7 rA rH r r r r r/ O s
r/ c @ s e Zd ZdZdd� Zdd� ZdS )�UploadResultSubscriberZuploadc C s0 |j j}| �|j�}d|j d |j }||fS �N�s3://�/)r8 � call_args�_get_src�fileobj�bucket�key�r4 r; rQ r r r r r rH � s z$UploadResultSubscriber._get_src_destc C s t |�S r r
�r4 rS r r r rR � s z UploadResultSubscriber._get_srcN)r, r- r. r3 rH rR r r r r rM � s rM c @ s e Zd Zdd� ZdS )� UploadStreamResultSubscriberc C s dS �N�-r rW r r r rR � s z%UploadStreamResultSubscriber._get_srcN)r, r- r. rR r r r r rX � s rX c @ s e Zd ZdZdd� Zdd� ZdS )�DownloadResultSubscriberZdownloadc C s0 |j j}d|j d |j }| �|j�}||fS rN )r8 rQ rT rU � _get_destrS rV r r r rH � s z&DownloadResultSubscriber._get_src_destc C s t |�S r r
rW r r r r\ � s z"DownloadResultSubscriber._get_destN)r, r- r. r3 rH r\ r r r r r[ � s r[ c @ s e Zd Zdd� ZdS )� DownloadStreamResultSubscriberc C s dS rY r rW r r r r\ � s z(DownloadStreamResultSubscriber._get_destN)r, r- r. r\ r r r r r] � s r] c @ s e Zd ZdZdd� ZdS )�CopyResultSubscriber�copyc C sB |j j}|j}d|d d |d }d|j d |j }||fS )NrO ZBucketrP ZKey)r8 rQ � copy_sourcerT rU )r4 r; rQ r` r r r r r rH � s
z"CopyResultSubscriber._get_src_destN�r, r- r. r3 rH r r r r r^ � s r^ c @ s e Zd ZdZdd� ZdS )�DeleteResultSubscriber�deletec C s$ |j j}d|j d |j }|d fS rN )r8 rQ rT rU )r4 r; rQ r r r r rH � s z$DeleteResultSubscriber._get_src_destNra r r r r rb � s rb c @ s e Zd ZdZdd� ZdS )�BaseResultHandlerz6Base handler class to be called in the ResultProcessorc C s t d��d S )Nz
__call__()rK �r4 �resultr r r �__call__� s zBaseResultHandler.__call__N)r, r- r. �__doc__rg r r r r rd � s rd c @ s� e Zd ZdZdd� Zdd� Zdd� Zdd � Zd
d � Zd d
� Z dd� Z
dd� Z dd� Z dd� Z
dd� Zdd� Zdd� Zd d � Zd S ) �ResultRecorderz@Records and track transfer statistics based on results receievedc C s� d| _ d| _d| _d| _d| _d| _d| _d| _d | _d | _ d| _
t t �| _
i | _t| jt| jt| jt| jt| jt| jt| jt | j i| _ d S �Nr ) r �bytes_failed_to_transfer�files_transferred� files_failed� files_warned�errors�expected_bytes_transferred�expected_files_transferred� final_expected_files_transferred�
start_time�bytes_transfer_speedr �int�_ongoing_progress�_ongoing_total_sizesr �_record_queued_resultr �_record_progress_resultr" �_record_success_resultr# �_record_failure_resultr �_record_warning_resultr& �_record_error_resultr' r) � _record_final_expected_files�_result_handler_map�r4 r r r r6 � s<
�zResultRecorder.__init__c C s | j | jkS r )rr rq r� r r r �expected_totals_are_final� s ��z(ResultRecorder.expected_totals_are_finalc C s | j �t|�| j�|d� dS )z0Record the result of an individual Result object�rf N)r �getr � _record_noopre r r r rg � s �zResultRecorder.__call__c C sT t |t�stdt|� ��g }|j|j|jfD ]}|d k r.|�t|�� q.d� |�S )NzdAny result using _get_ongoing_dict_key must subclass from BaseResult. Provided result is of type: %s�:)
rE r �
ValueErrorr r r r �appendr �join)r4 rf Z key_partsZresult_propertyr r r �_get_ongoing_dict_key� s
��z$ResultRecorder._get_ongoing_dict_keyc C s. | � |�}| j�|d�}| j�|d �}||fS rj )r� rv rJ rw )r4 rf � ongoing_key�total_progress�total_file_sizer r r � _pop_result_from_ongoing_dicts� s
z-ResultRecorder._pop_result_from_ongoing_dictsc K s d S r r �r4 r< r r r r� s zResultRecorder._record_noopc K sN | j d krt�� | _ |j}|| j| �|�< |r<| j|7 _| jd7 _d S �N� )rs r? r rw r� rp rq )r4 rf r< r r r r rx s
��z$ResultRecorder._record_queued_resultc K sZ |j }| �|� | j| �|� |7 < | j |7 _ |j| jkrV| j |j| j | _d S r )r �(_update_ongoing_transfer_size_if_unknownrv r� r! rs rt )r4 rf r< r r r r ry s
��
�z&ResultRecorder._record_progress_resultc C sh | � |�}| j| d krd|j}|jd k rT|| j|< | j| }|| }| j|7 _n| j|j7 _d S r )r� rw r rv rp r )r4 rf r� r Zongoing_progressZunaccounted_bytesr r r r� & s
z7ResultRecorder._update_ongoing_transfer_size_if_unknownc K s | � |� | jd7 _d S r� )r� rl �r4 rf r< r r r rz @ s
z%ResultRecorder._record_success_resultc K sL | � |�\}}|d k r,|| }| j|7 _| jd7 _| jd7 _d S r� )r� rk rm rl )r4 rf r< r� r� Z
progress_leftr r r r{ D s �z%ResultRecorder._record_failure_resultc K s | j d7 _ d S r� )rn r� r r r r| Q s z%ResultRecorder._record_warning_resultc K s | j d7 _ d S r� )ro r� r r r r} T s z#ResultRecorder._record_error_resultc K s |j | _d S r )r* rr r� r r r r~ W s z+ResultRecorder._record_final_expected_filesN)r, r- r. rh r6 r� rg r� r� r� rx ry r� rz r{ r| r} r~ r r r r ri � s
ri c @ s� e Zd ZdZdZdZde Zde ZdZde Z dZ
d Z d
Z d Z
d Zd
Zd8dd�Zdd� Zdd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Z d)d*� Z d+d,� Z d9d.d/�Z d0d1� Z d2d3� Z!d4d5� Z"d6d7� Z#dS ):�
ResultPrinterz#{remaining_files} file(s) remainingz~{expected_total}z (calculating...)zOCompleted {bytes_completed}/{expected_bytes_completed} ({transfer_speed}) with z)Completed {files_completed} file(s) with z${transfer_type}: {transfer_location}z (dryrun) z7{transfer_type} failed: {transfer_location} {exception}z {message}zfatal error: {exception}zcancelled: ctrl-c receivedz{src} to {dest}z{src}Nc C sv || _ || _| jdkr tj| _|| _| jdkr6tj| _d| _t| jt | j
t | j t
| jt| jt| jt| jt| ji| _dS )a� Prints status of ongoing transfer
:type result_recorder: ResultRecorder
:param result_recorder: The associated result recorder
:type out_file: file-like obj
:param out_file: Location to write progress and success statements.
By default, the location is sys.stdout.
:type error_file: file-like obj
:param error_file: Location to write warnings and errors.
By default, the location is sys.stderr.
Nr )�_result_recorder� _out_file�sys�stdout� _error_file�stderr�_progress_lengthr �_print_progressr" �_print_successr# �_print_failurer �_print_warningr& � _print_errorr' �
_print_ctrl_cr% �_print_dry_runr) �-_clear_progress_if_no_more_expected_transfersr )r4 �result_recorderZout_fileZ
error_filer r r r6 z s2
�zResultPrinter.__init__c C s | j �t|�| j�|d� dS )z<Print the progress of the ongoing transfer based on a resultr� N)r r� r � _print_noopre r r r rg � s �zResultPrinter.__call__c K s d S r r r� r r r r� � s zResultPrinter._print_noopc K s0 | j j|j| �|�d�}| �|�}| �|� d S �N)r �transfer_location)�DRY_RUN_FORMAT�formatr �_get_transfer_location�_adjust_statement_padding�_print_to_out_file)r4 rf r< � statementr r r r� � s �
z ResultPrinter._print_dry_runc K s8 | j j|j| �|�d�}| �|�}| �|� | �� d S r� )�SUCCESS_FORMATr� r r� r� r� �_redisplay_progress)r4 rf r< Zsuccess_statementr r r r� � s �
z ResultPrinter._print_successc K s< | j j|j| �|�|jd�}| �|�}| �|� | �� d S )N)r r� r$ )�FAILURE_FORMATr� r r� r$ r� �_print_to_error_filer� )r4 rf r< Zfailure_statementr r r r� � s �
z ResultPrinter._print_failurec K s0 | j j|jd�}| �|�}| �|� | �� d S )N)�message)�WARNING_FORMATr� r� r� r� r� )r4 rf r< Zwarning_statementr r r r� � s
z ResultPrinter._print_warningc K s | � | jj|jd�� d S )NrD )�_flush_error_statement� ERROR_FORMATr� r$ r� r r r r� � s �zResultPrinter._print_errorc K s | � | j� d S r )r� �
CTRL_C_MSGr� r r r r� � s zResultPrinter._print_ctrl_cc C s | � |�}| �|� d S r )r� r� )r4 Zerror_statementr r r r� � s
z$ResultPrinter._flush_error_statementc C s. |j d kr| jj|jd�S | jj|j|j d�S )N)r )r r )r � SRC_TRANSFER_LOCATION_FORMATr� r �!SRC_DEST_TRANSFER_LOCATION_FORMATre r r r r� � s
�z$ResultPrinter._get_transfer_locationc C s d| _ | �� d S rj )r� �_add_progress_if_neededr� r r r r� � s z!ResultPrinter._redisplay_progressc C s | � � r| �� d S r )�_has_remaining_progressr� r� r r r r� � s z%ResultPrinter._add_progress_if_neededc K s� | � t| jj| jj ��}| jjdkrrt| jj| jj �}| � t| jj��}t| jj �d }| j
j ||||d�}n| j j | jj|d�}| j�
� s�|| j7 }| j|dd�}t|�d | _| �|� d S )Nr z/s)�bytes_completed�expected_bytes_completed�transfer_speed�remaining_files)Zfiles_completedr� �
)� ending_charr� )�_get_expected_total�strr� rq rl rp r r rk rt �BYTE_PROGRESS_FORMATr� �FILE_PROGRESS_FORMATr� �_STILL_CALCULATING_TOTALSr� �lenr� r� )r4 r< r� r� r� r� Zprogress_statementr r r r� � sN �� ������� �
�z ResultPrinter._print_progressc C s | j �� s| jj|d�S |S )N)�expected_total)r� r� �_ESTIMATED_EXPECTED_TOTALr� )r4 r� r r r r� s
�z!ResultPrinter._get_expected_total�
c C s |� | jd�}|| S )N� )�ljustr� )r4 Zprint_statementr� r r r r� s z'ResultPrinter._adjust_statement_paddingc C s& | j �� sdS | j j}| j j}||kS )NT)r� r� rl rq )r4 ZactualZexpectedr r r r� s
z%ResultPrinter._has_remaining_progressc C s t || j� d S r )r
r� �r4 r� r r r r� " s z ResultPrinter._print_to_out_filec C s t || j� d S r )r
r� r� r r r r� % s z"ResultPrinter._print_to_error_filec K s$ | j r | �� s t| �d�| j� d S )N� )r� r� r
r� r� r� r r r r� ( s z;ResultPrinter._clear_progress_if_no_more_expected_transfers)NN)r� )$r, r- r. Z_FILES_REMAININGr� r� r� r� r� r� r� r� r� r� r� r� r6 rg r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r r� [ sR �������
"
/
r� c @ s e Zd ZdZdd� ZdS )�NoProgressResultPrinterz,A result printer that doesn't print progressc K s d S r r r� r r r r� / s z'NoProgressResultPrinter._print_progressN)r, r- r. rh r� r r r r r� - s r� c @ s e Zd ZdZdd� Zdd� ZdS )�OnlyShowErrorsResultPrinterz,A result printer that only prints out errorsc K s d S r r r� r r r r� 5 s z+OnlyShowErrorsResultPrinter._print_progressc K s d S r r r� r r r r� 8 s z*OnlyShowErrorsResultPrinter._print_successN)r, r- r. rh r� r� r r r r r� 3 s r� c @ s& e Zd Zddd�Zdd� Zdd� ZdS ) �ResultProcessorNc C s2 t j�| � || _|| _| jdkr(g | _d| _dS )aQ Thread to process results from result queue
This includes recording statistics and printing transfer status
:param result_queue: The result queue to process results from
:param result_handlers: A list of callables that take a result in as
a parameter to process the result for that handler.
NT)� threading�Threadr6 r0 �_result_handlers�_result_handlers_enabled)r4 r5 Zresult_handlersr r r r6 = s
zResultProcessor.__init__c C sf zH| j �d�}t|t�r&t�d� W qb| jr6| �|� t|t�rFd| _W q t j
k
r^ Y q X q d S )NTzSShutdown request received in result processing thread, shutting down result thread.F) r0 r� rE r+ �LOGGER�debugr� �_process_resultr& r ZEmptyre r r r �runM s
�
zResultProcessor.runc C sT | j D ]H}z ||� W q tk
rL } ztjd|||dd� W 5 d }~X Y qX qd S )Nz.Error processing result %s with handler %s: %sT��exc_info)r� � Exceptionr� r� )r4 rf Zresult_handlerrF r r r r� b s
�z ResultProcessor._process_result)N)r, r- r. r6 r� r� r r r r r� < s
r� c @ sD e Zd Zdd� Zdd� Zdd� Zdd� Zd d
� Zd d � Zd
d� Z dS )�CommandResultRecorderc C s || _ || _|| _dS )a� Records the result for an entire command
It will fully process all results in a result queue and determine
a CommandResult representing the entire command.
:type result_queue: queue.Queue
:param result_queue: The result queue in which results are placed on
and processed from
:type result_recorder: ResultRecorder
:param result_recorder: The result recorder to track the various
results sent through the result queue
:type result_processor: ResultProcessor
:param result_processor: The result processor to process results
placed on the queue
N)r5 r� �_result_processor)r4 r5 r� Zresult_processorr r r r6 m s z CommandResultRecorder.__init__c C s | j �� d S r )r� �startr� r r r r� � s zCommandResultRecorder.startc C s | j �t� � | j�� d S r )r5 r: r+ r� r� r� r r r �shutdown� s z CommandResultRecorder.shutdownc C s t | jj| jj | jj�S )z�Get the CommandResult representing the result of a command
:rtype: CommandResult
:returns: The CommandResult representing the total result from running
a particular command
)r( r� rm ro rn r� r r r �get_command_result� s �z(CommandResultRecorder.get_command_resultc C s | j �t|�� d S r )r5 r: r) )r4 Ztotalr r r �notify_total_submissions� s z.CommandResultRecorder.notify_total_submissionsc C s | � � | S r )r� r� r r r � __enter__� s z CommandResultRecorder.__enter__c G s> |r2t jd|dd� | j�t|d�� | �� dS | �� d S )Nz-Exception caught during command execution: %sTr� rD )r� r� r5 r: r& r� )r4 �exc_type� exc_value�argsr r r �__exit__� s �z CommandResultRecorder.__exit__N)
r, r- r. r6 r� r� r� r� r� r� r r r r r� l s r� )7Z
__future__r Zloggingr� r� r? � collectionsr r Zs3transfer.exceptionsr r Zs3transfer.subscribersr Z
awscli.compatr r Z awscli.customizations.s3.utilsr r Zawscli.customizations.utilsr
r r Z getLoggerr, r� r r r r r" r# r% r& r' r( r) �objectr+ r/ rM rX r[ r] r^ rb rd ri r� r� r� r� r� r� r r r r �<module>
sn
�� � �=
S 0