utils.py 文件源码

def search_shops_on_rus_forum(force=False):
    last_page = (MemberRus.objects.aggregate(Max('page_number')) and not force) or 1

    for i in range(last_page, 4219):
        page_url = 'http://prestadev.ru/forum/profile.php?u='+str(i)
        page = document_fromstring(urllib2.urlopen(page_url).read())
        messages = 0
        try:
            messages = int(page.cssselect('div.wttborder td strong')[2].text.strip())
        except:
            pass
        try:
            params = {'title': page.cssselect('#profilename')[0].text.strip(),
                  'messages': messages,
                  'page_number': i,
                  'home_page':page.cssselect('div.wttborder td.row1')[4]}
        except IndexError:
            continue
        member = MemberRus.objects.get_or_create(**params)[0]
        for link in page.cssselect('div.wgborder td.row1 a'):
            ShopLinkRus.objects.get_or_create(link=link.get('href'), member=member)