downloader.py 文件源码

def __save_mutual_fund_id_list_per_page(self, page_number):

        logger.info("__save_mutual_fund_id_list_per_page() function entry. page_number=%d" % page_number)

        # Add post parameters
        query_args = {"page": str(page_number),
                      "rp": str(self.mutualFundCountPerPage),
                      "sortname": "StandardName",
                      "sortorder": "asc",
                      "query": "",
                      "qtype": "StandardName",
                      "myFilter": "",
                      "FundIds": ""}
        encoded_args = urllib.urlencode(query_args)
        request = urllib2.Request(self.init_url,encoded_args)

        # Add headers
        request.add_header("Referer",
                           "http://www2.morningstar.ca/Components/FundTable/FundTable2.aspx?CobrandId=0&Culture=en-CA")
        request.add_header("User-Agent",
                           "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36")
        request.add_header("X-Requested-With",
                           "XMLHttpRequest")
        logger.debug("Http request: %s" % request.get_full_url())

        # Get http response and decode the json
        response = urllib2.urlopen(request)
        json_data = response.read()
        decoded_json = json.loads(json_data)

        if type(decoded_json[u"rows"]) == types.ListType:
            for row in decoded_json[u"rows"]:
                mutual_fund_id = row[u"id"]
                self.mutual_fund_id_list.append(mutual_fund_id)
                logger.debug("Save mutual fund id %s" % mutual_fund_id)