downloader.py 文件源码

def save_mutual_fund_info(self, mutual_fund_id):

        logger.info("__save_mutual_fund_info() function entry. {'mutual_fund_id': %s}" % mutual_fund_id)

        # Add GET parameters
        query_args = {"t": mutual_fund_id,
                      "region": "CAN",
                      "culture": "en-CA"}
        request = urllib2.Request(self.mutual_fund_url + "?" + urllib.urlencode(query_args))

        # Add headers
        request.add_header("Referer",
                           "http://www2.morningstar.ca/Components/FundTable/FundTable2.aspx?CobrandId=0&Culture=en-CA")
        request.add_header("User-Agent",
                           "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36")
        logger.debug("Http request: %s" % request.get_full_url())

        # Get http response and extract the url of mutual fund
        response = urllib2.urlopen(request)
        soup = BeautifulSoup(response.read(), 'html.parser')
        script_list = soup.find_all("script")
        pattern = r"t=[a-zA-Z0-9]+&region=CAN&culture=en-CA&cur=CAD&productCode=CAN"
        for script in script_list:
            match = re.search(pattern, unicode(script.string))
            if match:
                url_get_parameter_str = script.string[match.start():match.end()]

                # Split url GET parameter string
                get_parameter_str_list = url_get_parameter_str.split("&")
                get_parameter_dict = {}
                for get_parameter_str in get_parameter_str_list:
                    get_parameter_pair = get_parameter_str.split("=")
                    get_parameter_dict[get_parameter_pair[0]] = get_parameter_pair[1]

                # Create Mutual Fund Instance
                mutual_fund_inst = MutualFund()
                mutual_fund_inst.fund_id = mutual_fund_id

                # save Mutual Fund Head Portion
                self.__save_mutual_fund_head_portion(mutual_fund_inst, get_parameter_dict)
                if mutual_fund_inst.fund_name == "":
                    return None

                # save Mutual Fund Objective and Strategy Portion
                self.__save_mutual_fund_obj_strategy_portion(mutual_fund_inst, get_parameter_dict)

                # save Mutual Fund Performance Portion
                self.__save_mutual_fund_performance_portion(mutual_fund_inst, get_parameter_dict)

                # save Mutual Fund Annual Performance Portion
                #self.__save_mutual_fund_annual_performance_portion(mutual_fund_inst, get_parameter_dict)

                return mutual_fund_inst