downloader.py 文件源码

def save_all_mutual_fund_info(self, exporter):

        if os.path.exists(self.mutual_fund_id_list_file_name) and os.path.isfile(self.mutual_fund_id_list_file_name):
            logger.debug("Read mutual fund id list from %s" % self.mutual_fund_id_list_file_name)
            file_hdlr = open(self.mutual_fund_id_list_file_name, 'r')
            self.mutual_fund_id_list = json.load(file_hdlr)
            file_hdlr.close()
        else:
            logger.debug("%s file does not exist, need to read from Internet and write to file" % self.mutual_fund_id_list_file_name)
            page_count = self.__get_mutual_fund_page_count()
            for i in range(page_count):
                if i > 0:
                    self.__save_mutual_fund_id_list_per_page(i)
            self.__write_mutual_fund_id_list_to_file()
        print "Total %d Mutual Fund need to be downloaded." % len(self.mutual_fund_id_list)

        thread_inst_list = []
        for i in range(20):
            thread_inst = threading.Thread(name=str(i), target=thread_func, args=(self, i, exporter))
            thread_inst_list.append(thread_inst)
            thread_inst.setDaemon(True)
            thread_inst.start()
        for inst in thread_inst_list:
            inst.join()