item_queue_io.py 文件源码

def upload_queue_processor():
    """
    Implements a simple re-try mechanism for pending uploads
    :return:
    """
    while True:
        if upload_queue.not_empty:
            callable_up = upload_queue.get()  # blocks
            # TODO: pass in the actual item being updated/uploaded, so we can do more intelligent retry mechanisms
            was_list = isinstance(callable_up, list)
            last_modified_time = oauth = None
            if was_list:
                last_modified_time, callable_up, oauth = callable_up
            args = callable_up.args if isinstance(callable_up, partial) else None
            num_retries = 15
            for x in range(15):
                try:
                    ret_val = callable_up()
                    if was_list:
                        item = ret_val  # is the new/updated item
                        if isinstance(item, File):
                            client = Client(oauth)
                            file_obj = client.file(file_id=item.object_id).get()
                            redis_set(r_c, file_obj, last_modified_time, box_dir_path=BOX_DIR)
                    break
                except BoxAPIException as e:
                    crate_logger.debug('{the_args}, {the_trace}'.format(the_args=args,
                                                                        the_trace=traceback.format_exc()))
                    if e.status == 409:
                        crate_logger.debug('Apparently Box says this item already exists...'
                                           'and we were trying to create it. Need to handle this better. message: {}'.format(e.message))
                        break
                except (ConnectionError, BrokenPipeError, ProtocolError, ConnectionResetError):
                    time.sleep(3)
                    crate_logger.debug('{the_args}, {the_trace}'.format(the_args=args,
                                                                        the_trace=traceback.format_exc()))
                    if x >= num_retries - 1:
                        crate_logger.debug('Upload giving up on: {}'.format(callable_up))
                        # no immediate plans to do anything with this info, yet.
                        uploads_given_up_on.append(callable_up)
                except (TypeError, FileNotFoundError):
                    crate_logger.debug(traceback.format_exc())
                    break
            upload_queue.task_done()