def extract_images(filename):
"""Extract the images into a 4D uint8 numpy array [index, y, x, depth]."""
log = logger.get()
log.info('Extracting {}'.format(filename))
with gzip.open(filename) as bytestream:
magic = _read32(bytestream)
if magic != 2051:
raise ValueError(
'Invalid magic number %d in MNIST image file: %s' %
(magic, filename))
num_images = _read32(bytestream)
rows = _read32(bytestream)
cols = _read32(bytestream)
buf = bytestream.read(rows * cols * num_images)
data = numpy.frombuffer(buf, dtype=numpy.uint8)
data = data.reshape(num_images, rows, cols, 1)
return data
python类open()的实例源码
def extract_labels(filename, one_hot=False):
"""Extract the labels into a 1D uint8 numpy array [index]."""
log = logger.get()
log.info('Extracting {}'.format(filename))
with gzip.open(filename) as bytestream:
magic = _read32(bytestream)
if magic != 2049:
raise ValueError(
'Invalid magic number %d in MNIST label file: %s' %
(magic, filename))
num_items = _read32(bytestream)
buf = bytestream.read(num_items)
labels = numpy.frombuffer(buf, dtype=numpy.uint8)
if one_hot:
return dense_to_one_hot(labels)
return labels
def download_annotated(request, individual_id):
individual = get_object_or_404(Individual, pk=individual_id)
filepath = os.path.dirname(str(individual.vcf_file.name))
filename = os.path.basename(str(individual.vcf_file.name))
# path = settings.MEDIA_ROOT
# if filename.endswith('vcf.zip'):
# basename = filename.split('.vcf.zip')[0]
# else:
basename = filename.split('.vcf')[0]
fullpath = '%s/annotation.final.vcf.zip' % (filepath)
vcffile = open(fullpath, 'rb')
response = HttpResponse(vcffile, content_type='application/x-zip-compressed')
# # response['Content-Encoding'] = 'gzip'
response['Content-Disposition'] = 'attachment; filename=%s.annotated.mendelmd.vcf.zip' % basename
response['Content-Length'] = os.path.getsize(fullpath)
return response
def request_file(link, outfile, force_rerun_flag=False):
"""Download a file given a URL if the outfile does not exist already.
Args:
link (str): Link to download file.
outfile (str): Path to output file, will make a new file if it does not exist. Will not download if it does
exist, unless force_rerun_flag is True.
force_rerun_flag (bool): Flag to force re-downloading of the file if it exists already.
Returns:
str: Path to downloaded file.
"""
if force_rerun(flag=force_rerun_flag, outfile=outfile):
req = requests.get(link)
if req.status_code == 200:
with open(outfile, 'w') as f:
f.write(req.text)
log.debug('Loaded and saved {} to {}'.format(link, outfile))
else:
log.error('{}: request error {}'.format(link, req.status_code))
return outfile
def write_torque_script(command, outfile, walltime, queue, name, out, err, print_exec=True):
with open(outfile, 'w') as script:
script.write('#PBS -l walltime={}\n'.format(walltime))
script.write('#PBS -q regular\n')
script.write('#PBS -N {}\n'.format(name))
script.write('#PBS -o {}.o\n'.format(out))
script.write('#PBS -e {}.e\n'.format(err))
script.write('cd ${PBS_O_WORKDIR}\n')
script.write(command)
os.chmod(outfile, 0o755)
if print_exec:
print('qsub {};'.format(outfile))
return outfile
def save(self, filename):
"""
Save the state of this network to a pickle file on disk.
:param filename: Save the parameters of this network to a pickle file at the named path. If this name ends in
".gz" then the output will automatically be gzipped; otherwise the output will be a "raw" pickle.
:return: None
"""
state = dict([('class', self.__class__.__name__), ('network', self.__str__())])
for layer in self.layers:
key = '{}-values'.format(layer.layerNum)
state[key] = [p.get_value() for p in layer.params]
opener = gzip.open if filename.lower().endswith('.gz') else open
handle = opener(filename, 'wb')
cPickle.dump(state, handle, -1)
handle.close()
print 'Saved model parameter to {}'.format(filename)
def append(self, text):
try:
from Naked.toolshed.system import file_exists
if not file_exists(self.filepath): #confirm that file exists, if not raise IOError (assuming that developer expected existing file if using append)
raise IOError("The file specified for the text append does not exist (Naked.toolshed.file.py:append).")
with open(self.filepath, 'a') as appender:
appender.write(text)
except UnicodeEncodeError as ue:
self.append_utf8(text) #try writing as utf-8
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: Unable to append text to the file with the append() method (Naked.toolshed.file.py).")
raise e
#------------------------------------------------------------------------------
# [ append_utf8 method ]
# Text writer that appends text to existing file with utf-8 encoding
# Tests: test_IO.py :: test_file_utf8_readwrite_append
#------------------------------------------------------------------------------
def append_utf8(self, text):
try:
from Naked.toolshed.system import file_exists
if not file_exists(self.filepath):
raise IOError("The file specified for the text append does not exist (Naked.toolshed.file.py:append_utf8).")
import codecs
import unicodedata
norm_text = unicodedata.normalize('NFKD', text) # NKFD normalization of the unicode data before write
with codecs.open(self.filepath, mode='a', encoding="utf_8") as appender:
appender.write(norm_text)
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: Unable to append text to the file with the append_utf8 method (Naked.toolshed.file.py).")
raise e
#------------------------------------------------------------------------------
# [ gzip method (writer) ]
# writes data to gzip compressed file
# Note: adds .gz extension to filename if user did not specify it in the FileWriter class constructor
# Note: uses compresslevel = 6 as default to balance speed and compression level (which in general is not significantly less than 9)
# Tests: test_IO.py :: test_file_gzip_ascii_readwrite, test_file_gzip_utf8_readwrite,
# test_file_gzip_utf8_readwrite_explicit_decode
#------------------------------------------------------------------------------
def gzip(self, text, compression_level=6):
try:
import gzip
if not self.filepath.endswith(".gz"):
self.filepath = self.filepath + ".gz"
with gzip.open(self.filepath, 'wb', compresslevel=compression_level) as gzip_writer:
gzip_writer.write(text)
except UnicodeEncodeError as ue:
import unicodedata
norm_text = unicodedata.normalize('NFKD', text) # NKFD normalization of the unicode data before write
import codecs
binary_data = codecs.encode(norm_text, "utf_8")
with gzip.open(self.filepath, 'wb', compresslevel=compression_level) as gzip_writer:
gzip_writer.write(binary_data)
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: unable to gzip compress the file with the gzip method (Naked.toolshed.file.py).")
raise e
#------------------------------------------------------------------------------
# [ write method ]
# Universal text file writer that writes by system default or utf-8 encoded unicode if throws UnicdeEncodeError
# Tests: test_IO.py :: test_file_ascii_readwrite, test_file_ascii_readwrite_missing_file,
# test_file_utf8_write_raises_unicodeerror
#------------------------------------------------------------------------------
def write(self, text):
try:
with open(self.filepath, 'wt') as writer:
writer.write(text)
except UnicodeEncodeError as ue:
self.write_utf8(text) # attempt to write with utf-8 encoding
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: Unable to write to requested file with the write() method (Naked.toolshed.file.py).")
raise e
#------------------------------------------------------------------------------
# [ write_as method ]
# text file writer that uses developer specified text encoding
# Tests: test_IO.py :: test_file_utf8_readas_writeas
#------------------------------------------------------------------------------
def write_as(self, text, the_encoding=""):
try:
if the_encoding == "": #if the developer did not include the encoding type, raise an exception
raise RuntimeError("The text encoding was not specified as an argument to the write_as() method (Naked.toolshed.file.py:write_as).")
import codecs
with codecs.open(self.filepath, encoding=the_encoding, mode='w') as f:
f.write(text)
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: unable to write file with the specified encoding using the write_as() method (Naked.toolshed.file.py).")
raise e
#------------------------------------------------------------------------------
# [ write_bin method ]
# binary data file writer
# Tests: test_IO.py :: test_file_bin_readwrite
#------------------------------------------------------------------------------
def safe_write(self, text):
import os.path
if not os.path.exists(self.filepath): # if the file does not exist, then can write
try:
with open(self.filepath, 'wt') as writer:
writer.write(text)
return True
except UnicodeEncodeError as ue:
self.write_utf8(text)
return True
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: Unable to write to requested file with the safe_write() method (Naked.toolshed.file.py).")
raise e
else:
return False # if file exists, do not write and return False
#------------------------------------------------------------------------------
# [ safe_write_bin method ]
# Binary data file writer that will NOT overwrite existing file at the requested filepath
# returns boolean indicator for success of write based upon test for existence of file (False = write failed because file exists)
#------------------------------------------------------------------------------
def safe_write_bin(self, file_data):
try:
import os.path
if not os.path.exists(self.filepath):
with open(self.filepath, 'wb') as writer:
writer.write(file_data)
return True
else:
return False
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: Unable to write to requested file with the safe_write_bin() method (Naked.toolshed.file.py).")
raise e
#------------------------------------------------------------------------------
# [ write_utf8 method ]
# Text file writer with explicit UTF-8 text encoding
# uses filepath from class constructor
# requires text to passed as a method parameter
# Tests: test_IO.py :: test_file_utf8_readwrite, test_file_utf8_readwrite_raises_unicodeerror
#------------------------------------------------------------------------------
def write_utf8(self, text):
try:
import codecs
f = codecs.open(self.filepath, encoding='utf_8', mode='w')
except IOError as ioe:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: Unable to open file for write with the write_utf8() method (Naked.toolshed.file.py).")
raise ioe
try:
import unicodedata
norm_text = unicodedata.normalize('NFKD', text) # NKFD normalization of the unicode data before write
f.write(norm_text)
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: Unable to write UTF-8 encoded text to file with the write_utf8() method (Naked.toolshed.file.py).")
raise e
finally:
f.close()
#------------------------------------------------------------------------------
# [ FileReader class ]
# reads data from local files
# filename assigned in constructor (inherited from IO class interface)
#------------------------------------------------------------------------------
def read_bin(self):
try:
with open(self.filepath, 'rb') as bin_reader:
data = bin_reader.read()
return data
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: Unable to read the binary data from the file with the read_bin method (Naked.toolshed.file.py).")
raise e
#------------------------------------------------------------------------------
# [ read_as method ] (string with developer specified text encoding)
# Text file reader with developer specified text encoding
# returns file contents in developer specified text encoding
# Tests: test_IO.py :: test_file_utf8_readas_writeas, test_file_readas_missing_file
#------------------------------------------------------------------------------
def read_as(self, the_encoding):
try:
if the_encoding == "":
raise RuntimeError("The text file encoding was not specified as an argument to the read_as method (Naked.toolshed.file.py:read_as).")
import codecs
with codecs.open(self.filepath, encoding=the_encoding, mode='r') as f:
data = f.read()
return data
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: Unable to read the file with the developer specified text encoding with the read_as method (Naked.toolshed.file.py).")
raise e
#------------------------------------------------------------------------------
# [ readlines method ] (list of strings)
# Read text from file line by line, uses utf8 encoding by default
# returns list of utf8 encoded file lines as strings
# Tests: test_IO.py :: test_file_readlines, test_file_readlines_missing_file
#------------------------------------------------------------------------------
def readlines_utf8(self):
try:
import codecs
with codecs.open(self.filepath, encoding='utf-8', mode='r') as uni_reader:
modified_text_list = []
for line in uni_reader:
import unicodedata
norm_line = unicodedata.normalize('NFKD', line) # NKFD normalization of the unicode data before use
modified_text_list.append(norm_line)
return modified_text_list
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: unable to read lines in the unicode file with the readlines_utf8 method (Naked.toolshed.file.py)")
raise e
#------------------------------------------------------------------------------
# [ read_gzip ] (byte string)
# reads data from a gzip compressed file
# returns the decompressed binary data from the file
# Note: if decompressing unicode file, set encoding="utf-8"
# Tests: test_IO.py :: test_file_gzip_ascii_readwrite, test_file_gzip_utf8_readwrite,
# test_file_read_gzip_missing_file
#------------------------------------------------------------------------------
def read_utf8(self):
try:
import codecs
f = codecs.open(self.filepath, encoding='utf_8', mode='r')
except IOError as ioe:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: Unable to open file for read with read_utf8() method (Naked.toolshed.file.py).")
raise ioe
try:
textstring = f.read()
import unicodedata
norm_text = unicodedata.normalize('NFKD', textstring) # NKFD normalization of the unicode data before returns
return norm_text
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: Unable to read the file with UTF-8 encoding using the read_utf8() method (Naked.toolshed.file.py).")
raise e
finally:
f.close()
def append(self, text):
try:
from Naked.toolshed.system import file_exists
if not file_exists(self.filepath): #confirm that file exists, if not raise IOError (assuming that developer expected existing file if using append)
raise IOError("The file specified for the text append does not exist (Naked.toolshed.file.py:append).")
with open(self.filepath, 'a') as appender:
appender.write(text)
except UnicodeEncodeError as ue:
self.append_utf8(text) #try writing as utf-8
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: Unable to append text to the file with the append() method (Naked.toolshed.file.py).")
raise e
#------------------------------------------------------------------------------
# [ append_utf8 method ]
# Text writer that appends text to existing file with utf-8 encoding
# Tests: test_IO.py :: test_file_utf8_readwrite_append
#------------------------------------------------------------------------------
def append_utf8(self, text):
try:
from Naked.toolshed.system import file_exists
if not file_exists(self.filepath):
raise IOError("The file specified for the text append does not exist (Naked.toolshed.file.py:append_utf8).")
import codecs
import unicodedata
norm_text = unicodedata.normalize('NFKD', text) # NKFD normalization of the unicode data before write
with codecs.open(self.filepath, mode='a', encoding="utf_8") as appender:
appender.write(norm_text)
except Exception as e:
if DEBUG_FLAG:
sys.stderr.write("Naked Framework Error: Unable to append text to the file with the append_utf8 method (Naked.toolshed.file.py).")
raise e
#------------------------------------------------------------------------------
# [ gzip method (writer) ]
# writes data to gzip compressed file
# Note: adds .gz extension to filename if user did not specify it in the FileWriter class constructor
# Note: uses compresslevel = 6 as default to balance speed and compression level (which in general is not significantly less than 9)
# Tests: test_IO.py :: test_file_gzip_ascii_readwrite, test_file_gzip_utf8_readwrite,
# test_file_gzip_utf8_readwrite_explicit_decode
#------------------------------------------------------------------------------
