entity.py 文件源码

def retrieve_data(self, session):
        """
        Retrieve information about entity using an existing session.
        :param session: Requests session to use for data retrieval.
        :return: True if data about entity has been successfully retrieved and no filter callback excluded this entity,
            False otherwise.
        """

        try:
            logger.info("Retrieving data for entity " + str(self) + "...")

            # execute pre_request_callbacks
            for callback in self.configuration.pre_request_callbacks:
                callback(self)

            # reduce request frequency as configured
            delay = randint(self.configuration.delay_min,
                            self.configuration.delay_max)  # delay between requests in milliseconds
            time.sleep(delay / 1000)  # sleep for delay ms to prevent getting blocked

            # retrieve data
            if len(self.configuration.headers) > 0:
                response = session.get(self.uri, headers=self.configuration.headers)
            else:
                response = session.get(self.uri)

            if response.ok:
                logger.info("Successfully retrieved data for entity " + str(self) + ".")

                if self.configuration.raw_download:
                    # raw download
                    self.output_parameters[self.configuration.raw_parameter] = response.content
                else:
                    # JSON API call
                    # deserialize JSON string
                    json_response = json.loads(response.text)
                    # extract parameters according to parameter mapping
                    self._extract_output_parameters(json_response)

                # execute post_request_callbacks
                for callback in self.configuration.post_request_callbacks:
                    result = callback(self)
                    # check if callback implements filter
                    if isinstance(result, bool):
                        if not result:
                            logger.info("Entity removed because of filter callback " + str(callback) + ": " + str(self))
                            return False

                    return True

            else:
                logger.error("Error " + str(response.status_code) + ": Could not retrieve data for entity " + str(self)
                             + ". Response: " + str(response.content))
                return False

        except (gaierror,
                ConnectionError,
                MaxRetryError,
                NewConnectionError):
            logger.error("An error occurred while retrieving data for entity  " + str(self) + ".")