downloader.py 文件源码

def __get_mutual_fund_page_count(self):

        logger.info("__get_mutual_fund_page_count() function entry")

        # Add post parameters
        query_args = {"page": "1",
                      "rp": str(self.mutualFundCountPerPage),
                      "sortname": "StandardName",
                      "sortorder": "asc",
                      "query": "",
                      "qtype": "StandardName",
                      "myFilter": "",
                      "FundIds": ""}
        encoded_args = urllib.urlencode(query_args)
        request = urllib2.Request(self.init_url,encoded_args)

        # Add headers
        request.add_header("Referer",
                           "http://www2.morningstar.ca/Components/FundTable/FundTable2.aspx?CobrandId=0&Culture=en-CA")
        request.add_header("User-Agent",
                           "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36")
        request.add_header("X-Requested-With",
                           "XMLHttpRequest")

        logger.debug("Http request: %s" % request.get_full_url())

        # Get http response and decode the json
        response = urllib2.urlopen(request)
        json_data = response.read()
        decoded_json = json.loads(json_data)
        self.totalMutualFundCount = int(decoded_json[u"total"])
        logger.debug("Total mutual fund count is %d" % self.totalMutualFundCount)

        return self.totalMutualFundCount / self.mutualFundCountPerPage