hive_hooks.py 文件源码

def to_csv(
            self,
            hql,
            csv_filepath,
            schema='default',
            delimiter=',',
            lineterminator='\r\n',
            output_header=True,
            fetch_size=1000):
        schema = schema or 'default'
        with self.get_conn(schema) as conn:
            with conn.cursor() as cur:
                self.log.info("Running query: %s", hql)
                cur.execute(hql)
                schema = cur.description
                with open(csv_filepath, 'wb') as f:
                    writer = csv.writer(f,
                                        delimiter=delimiter,
                                        lineterminator=lineterminator,
                                        encoding='utf-8')
                    if output_header:
                        writer.writerow([c[0] for c in cur.description])
                    i = 0
                    while True:
                        rows = [row for row in cur.fetchmany(fetch_size) if row]
                        if not rows:
                            break

                        writer.writerows(rows)
                        i += len(rows)
                        self.log.info("Written %s rows so far.", i)
                    self.log.info("Done. Loaded a total of %s rows.", i)