data_import.py 文件源码

def import_people(url):
    r = requests.get(url)
    if r.status_code != 200:
        sys.stderr.write('Error retrieving %s:\n%s\n' % (url, r))
        return

    total_created = 0

    soup = bs4.BeautifulSoup(r.text, 'html.parser')
    for row in soup.table.find_all('tr'):
        columns = row.find_all('td')
        if len(columns) != 4:
            continue

        (name, room, phone, email) = [ c.text for c in columns ]
        if not email.endswith('mun.ca'):
            continue

        name = name.split(',')[0].strip()
        email = email.replace('[at]', '@').strip()

        try:
            _, created = db.Person.get_or_create(name = name, email = email)
            total_created += created

        except peewee.IntegrityError, e:
            sys.stderr.write("""error: failure to create person '%s' (%s)
  note: %s
  note: existing people with same email:
%s
""" % (
                    name, email, e,
                    ''.join([
                        "    - '%s' (%s)\n" % (p.name, p.email) for p in
                            db.Person.select().where(db.Person.email == email)
                    ])
            ))

    print('Imported details of %d individuals.' % total_created)