def parse_sitemap(content):
if not isinstance(content, six.text_type):
content = content.decode('utf-8')
urlset_match = re.search(
r'<urlset[^>]*>(?P<urls>[\s\S]*)</urlset>', content
)
if urlset_match:
results = []
urlset_content = urlset_match.groupdict()['urls']
for url_content in re.findall(r'<url>([\s\S]+)</url>', urlset_content):
results.append(
dict(
re.findall(r'<([^>]+)>([^<]*)</[^>]+>', url_content)
)
)
else:
results = None
return results
python类text_type()的实例源码
def test_chart_view_get(self):
ChartViewSubClass = type('ChartViewSubClass', (ChartView, ), {
'chart_instance': LineChart()
})
chart_view = ChartViewSubClass()
request_factory = RequestFactory()
request = request_factory.get('/test-url')
response = chart_view.get(request)
self.assertEquals(response.status_code, 200)
charset = getattr(response, 'charset', 'utf-8')
content = response.content.decode(charset)
data = json.loads(content)
self.assertIn('data', data)
self.assertIn('options', data)
self.assertIn('type', data)
self.assertTrue(isinstance(data['data'], dict))
self.assertTrue(isinstance(data['options'], dict))
self.assertTrue(isinstance(data['type'], (six.string_types, six.text_type)))
self.assertIn(data['type'], ['bar', 'line', 'radar', 'polarArea', 'pie', 'bubble'])
self.assertIn('title', data['options'])
def test_chart_view_get(self):
ChartViewSubClass = type('ChartViewSubClass', (ChartView, ), {
'chart_instance': LineChart()
})
chart_view = ChartViewSubClass()
request_factory = RequestFactory()
request = request_factory.get('/test-url')
response = chart_view.get(request)
self.assertEquals(response.status_code, 200)
charset = getattr(response, 'charset', 'utf-8')
content = response.content.decode(charset)
data = json.loads(content)
self.assertIn('data', data)
self.assertIn('options', data)
self.assertIn('type', data)
self.assertTrue(isinstance(data['data'], dict))
self.assertTrue(isinstance(data['options'], dict))
self.assertTrue(isinstance(data['type'], (six.string_types, six.text_type)))
self.assertIn(data['type'], ['bar', 'line', 'radar', 'polarArea', 'pie', 'bubble'])
self.assertIn('title', data['options'])
def deconstruct(self):
name, path, args, kwargs = super(CollectionField, self).deconstruct()
if self.collection_type is not list:
kwargs['collection_type'] = self.collection_type
if self.item_type is not six.text_type:
kwargs['item_type'] = self.item_type
if self.sort:
kwargs['sort'] = True
if self.unique_items is not None:
kwargs['unique_items'] = self.unique_items
if self.max_items is not None:
kwargs['max_items'] = self.max_items
if self.delimiter != '|':
kwargs['delimiter'] = self.delimiter
if kwargs.get('max_length') == 1024:
del kwargs['max_length']
if kwargs.get('default') is self.collection_type:
del kwargs['default']
return name, path, args, kwargs
def prepare_value(self, value):
"""Converts value to list of string for widget usage"""
if not isinstance(value, choicelist):
value = value or ()
value = self.collection_type(
self.item_type(item) for item in value
)
if isinstance(value, set) or not self.unique_items:
collection = value
else:
collection = []
# Remove the duplicates while keeping order:
[
collection.append(item)
for item in value if item not in collection
]
if self.sort:
collection = sorted(collection)
value = choicelist(
six.text_type(item) for item in collection
)
return value
def _resize(original_size, index, divisor=0, padding=0,
keep_aspect_ratio=False):
if isinstance(original_size, six.text_type):
m = RE_SIZE.match(original_size)
if m:
original_size = (int(m.group(1)), int(m.group(2)))
else:
return original_size
else:
try:
original_size = (int(original_size[0]), int(original_size[1]))
except (TypeError, ValueError):
return original_size
try:
padding = int(padding)
divisor = int(divisor)
except (TypeError, ValueError):
return original_size
# Re-calculate size
new_x, new_y = _recalculate_size(original_size, index, divisor=divisor,
padding=padding,
keep_aspect_ratio=keep_aspect_ratio)
return (new_x, new_y)
def regex(self):
"""
Returns a compiled regular expression, depending upon the activated
language-code.
"""
language_code = get_language()
if language_code not in self._regex_dict:
if isinstance(self._regex, six.string_types):
regex = self._regex
else:
regex = force_text(self._regex)
try:
compiled_regex = re.compile(regex, re.UNICODE)
except re.error as e:
raise ImproperlyConfigured(
'"%s" is not a valid regular expression: %s' %
(regex, six.text_type(e)))
self._regex_dict[language_code] = compiled_regex
return self._regex_dict[language_code]
def to_python(self, value):
if not value:
return {}
if not isinstance(value, dict):
try:
value = json.loads(value)
except ValueError:
raise ValidationError(
self.error_messages['invalid_json'],
code='invalid_json',
)
if not isinstance(value, dict):
raise ValidationError(
self.error_messages['invalid_format'],
code='invalid_format',
)
# Cast everything to strings for ease.
for key, val in value.items():
value[key] = six.text_type(val)
return value
def url_params_from_lookup_dict(lookups):
"""
Converts the type of lookups specified in a ForeignKey limit_choices_to
attribute to a dictionary of query parameters
"""
params = {}
if lookups and hasattr(lookups, 'items'):
items = []
for k, v in lookups.items():
if callable(v):
v = v()
if isinstance(v, (tuple, list)):
v = ','.join(str(x) for x in v)
elif isinstance(v, bool):
# See django.db.fields.BooleanField.get_prep_lookup
v = ('0', '1')[v]
else:
v = six.text_type(v)
items.append((k, v))
params.update(dict(items))
return params
def make_bytes(self, value):
"""Turn a value into a bytestring encoded in the output charset."""
# Per PEP 3333, this response body must be bytes. To avoid returning
# an instance of a subclass, this function returns `bytes(value)`.
# This doesn't make a copy when `value` already contains bytes.
# Handle string types -- we can't rely on force_bytes here because:
# - under Python 3 it attempts str conversion first
# - when self._charset != 'utf-8' it re-encodes the content
if isinstance(value, bytes):
return bytes(value)
if isinstance(value, six.text_type):
return bytes(value.encode(self.charset))
# Handle non-string types (#16494)
return force_bytes(value, self.charset)
# These methods partially implement the file-like object interface.
# See https://docs.python.org/3/library/io.html#io.IOBase
# The WSGI server must call this method upon completion of the request.
# See http://blog.dscpl.com.au/2012/10/obligations-for-calling-close-on.html
def effective_default(self, field):
"""
Returns a field's effective database default value
"""
if field.has_default():
default = field.get_default()
elif not field.null and field.blank and field.empty_strings_allowed:
if field.get_internal_type() == "BinaryField":
default = six.binary_type()
else:
default = six.text_type()
else:
default = None
# If it's a callable, call it
if six.callable(default):
default = default()
# Run it through the field's get_db_prep_save method so we can send it
# to the database.
default = field.get_db_prep_save(default, self.connection)
return default
def date_error_message(self, lookup_type, field_name, unique_for):
opts = self._meta
field = opts.get_field(field_name)
return ValidationError(
message=field.error_messages['unique_for_date'],
code='unique_for_date',
params={
'model': self,
'model_name': six.text_type(capfirst(opts.verbose_name)),
'lookup_type': lookup_type,
'field': field_name,
'field_label': six.text_type(capfirst(field.verbose_name)),
'date_field': unique_for,
'date_field_label': six.text_type(capfirst(opts.get_field(unique_for).verbose_name)),
}
)
def assertHTMLEqual(self, html1, html2, msg=None):
"""
Asserts that two HTML snippets are semantically the same.
Whitespace in most cases is ignored, and attribute ordering is not
significant. The passed-in arguments must be valid HTML.
"""
dom1 = assert_and_parse_html(self, html1, msg,
'First argument is not valid HTML:')
dom2 = assert_and_parse_html(self, html2, msg,
'Second argument is not valid HTML:')
if dom1 != dom2:
standardMsg = '%s != %s' % (
safe_repr(dom1, True), safe_repr(dom2, True))
diff = ('\n' + '\n'.join(difflib.ndiff(
six.text_type(dom1).splitlines(),
six.text_type(dom2).splitlines())))
standardMsg = self._truncateMessage(standardMsg, diff)
self.fail(self._formatMessage(msg, standardMsg))
def assertXMLEqual(self, xml1, xml2, msg=None):
"""
Asserts that two XML snippets are semantically the same.
Whitespace in most cases is ignored, and attribute ordering is not
significant. The passed-in arguments must be valid XML.
"""
try:
result = compare_xml(xml1, xml2)
except Exception as e:
standardMsg = 'First or second argument is not valid XML\n%s' % e
self.fail(self._formatMessage(msg, standardMsg))
else:
if not result:
standardMsg = '%s != %s' % (safe_repr(xml1, True), safe_repr(xml2, True))
diff = ('\n' + '\n'.join(
difflib.ndiff(
six.text_type(xml1).splitlines(),
six.text_type(xml2).splitlines(),
)
))
standardMsg = self._truncateMessage(standardMsg, diff)
self.fail(self._formatMessage(msg, standardMsg))
def localize(value, use_l10n=None):
"""
Checks if value is a localizable type (date, number...) and returns it
formatted as a string using current locale format.
If use_l10n is provided and is not None, that will force the value to
be localized (or not), overriding the value of settings.USE_L10N.
"""
if isinstance(value, bool):
return mark_safe(six.text_type(value))
elif isinstance(value, (decimal.Decimal, float) + six.integer_types):
return number_format(value, use_l10n=use_l10n)
elif isinstance(value, datetime.datetime):
return date_format(value, 'DATETIME_FORMAT', use_l10n=use_l10n)
elif isinstance(value, datetime.date):
return date_format(value, use_l10n=use_l10n)
elif isinstance(value, datetime.time):
return time_format(value, 'TIME_FORMAT', use_l10n=use_l10n)
else:
return value
def T(self):
"""
Time zone of this machine; e.g. 'EST' or 'MDT'.
If timezone information is not available, this method returns
an empty string.
"""
if not self.timezone:
return ""
name = None
try:
name = self.timezone.tzname(self.data)
except Exception:
# pytz raises AmbiguousTimeError during the autumn DST change.
# This happens mainly when __init__ receives a naive datetime
# and sets self.timezone = get_default_timezone().
pass
if name is None:
name = self.format('O')
return six.text_type(name)
def url_params_from_lookup_dict(lookups):
"""
Converts the type of lookups specified in a ForeignKey limit_choices_to
attribute to a dictionary of query parameters
"""
params = {}
if lookups and hasattr(lookups, 'items'):
items = []
for k, v in lookups.items():
if callable(v):
v = v()
if isinstance(v, (tuple, list)):
v = ','.join(str(x) for x in v)
elif isinstance(v, bool):
v = ('0', '1')[v]
else:
v = six.text_type(v)
items.append((k, v))
params.update(dict(items))
return params
def _process_filter(self, field_attname, lookup, value):
# Get the field
field = self._get_filterable_field(field_attname)
if field is None:
raise FieldError(
'Cannot filter search results with field "' + field_attname + '". Please add index.FilterField(\'' +
field_attname + '\') to ' + self.queryset.model.__name__ + '.search_fields.'
)
# Process the lookup
result = self._process_lookup(field, lookup, value)
if result is None:
raise FilterError(
'Could not apply filter on search results: "' + field_attname + '__' +
lookup + ' = ' + text_type(value) + '". Lookup "' + lookup + '"" not recognised.'
)
return result
def product_chosen(request, pk):
product = get_object_or_404(Product, pk=pk)
product_json = json.dumps({
'id': product.pk,
'string': text_type(product),
'edit_link': reverse(
'dashboard:catalogue-product', kwargs={'pk': product.pk})
})
return render_modal_workflow(
request,
None, 'oscar_wagtail/chooser/product_chosen.js',
{
'product_json': product_json,
}
)
def to_python(self, value):
if self.disabled:
return value
if value in self.empty_values:
return None
elif isinstance(value, (list, dict, int, float, JSONString)):
return value
try:
converted = json.loads(value)
except ValueError:
raise forms.ValidationError(
self.error_messages['invalid'],
code='invalid',
params={'value': value},
)
if isinstance(converted, six.text_type):
return JSONString(converted)
else:
return converted
def url_params_from_lookup_dict(lookups):
"""
Converts the type of lookups specified in a ForeignKey limit_choices_to
attribute to a dictionary of query parameters
"""
params = {}
if lookups and hasattr(lookups, 'items'):
items = []
for k, v in lookups.items():
if callable(v):
v = v()
if isinstance(v, (tuple, list)):
v = ','.join(str(x) for x in v)
elif isinstance(v, bool):
v = ('0', '1')[v]
else:
v = six.text_type(v)
items.append((k, v))
params.update(dict(items))
return params
def to_python(self, value):
if self.disabled:
return value
if value in self.empty_values:
return None
elif isinstance(value, (list, dict, int, float, JSONString)):
return value
try:
converted = json.loads(value)
except ValueError:
raise forms.ValidationError(
self.error_messages['invalid'],
code='invalid',
params={'value': value},
)
if isinstance(converted, six.text_type):
return JSONString(converted)
else:
return converted
def http_quote(string):
"""
Given a unicode string, will do its dandiest to give you back a
valid ascii charset string you can use in, say, http headers and the
like.
"""
if isinstance(string, six.text_type):
try:
import unidecode
except ImportError:
pass
else:
string = unidecode.unidecode(string)
string = string.encode('ascii', 'replace')
# Wrap in double-quotes for ; , and the like
string = string.replace(b'\\', b'\\\\').replace(b'"', b'\\"')
return '"{0!s}"'.format(string.decode())
def _process_attr(self, attr):
params = re.findall(
r'({result=(?P<name>\w+):\$\.(?P<value>[a-zA-Z0-9.*]+)})', attr
)
if not params:
return attr
for url_param in params:
if url_param[1] not in self.named_responses:
raise ValidationError('Named request {} is missing'.format(url_param[1]))
result = get_attribute(
self.named_responses[url_param[1]]['_data'],
url_param[2].split('.')
)
if isinstance(result, list):
result = ','.join(map(six.text_type, result))
if attr == url_param[0]:
attr = result
else:
attr = attr.replace(url_param[0], str(result))
return attr
def __init__(self, choices, **kwargs):
self.grouped_choices = to_choices_dict(choices)
self.choices = flatten_choices_dict(self.grouped_choices)
self.html_cutoff = kwargs.pop('html_cutoff', self.html_cutoff)
self.html_cutoff_text = kwargs.pop('html_cutoff_text', self.html_cutoff_text)
# Map the string representation of choices to the underlying value.
# Allows us to deal with eg. integer choices while supporting either
# integer or string input, but still get the correct datatype out.
self.choice_strings_to_values = {
six.text_type(key): key for key in self.choices.keys()
}
self.allow_blank = kwargs.pop('allow_blank', False)
super(ChoiceField, self).__init__(**kwargs)
def render_field(self, field, parent_style):
if isinstance(field._field, serializers.HiddenField):
return ''
style = dict(self.default_style[field])
style.update(field.style)
if 'template_pack' not in style:
style['template_pack'] = parent_style.get('template_pack', self.template_pack)
style['renderer'] = self
# Get a clone of the field with text-only value representation.
field = field.as_form_field()
if style.get('input_type') == 'datetime-local' and isinstance(field.value, six.text_type):
field.value = field.value.rstrip('Z')
if 'template' in style:
template_name = style['template']
else:
template_name = style['template_pack'].strip('/') + '/' + style['base_template']
template = loader.get_template(template_name)
context = {'field': field, 'style': style}
return template_render(template, context)
def parse(self, stream, media_type=None, parser_context=None):
"""
Parses the incoming bytestream as a multipart encoded form,
and returns a DataAndFiles object.
`.data` will be a `QueryDict` containing all the form parameters.
`.files` will be a `QueryDict` containing all the form files.
"""
parser_context = parser_context or {}
request = parser_context['request']
encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET)
meta = request.META.copy()
meta['CONTENT_TYPE'] = media_type
upload_handlers = request.upload_handlers
try:
parser = DjangoMultiPartParser(meta, stream, upload_handlers, encoding)
data, files = parser.parse()
return DataAndFiles(data, files)
except MultiPartParserError as exc:
raise ParseError('Multipart form parse error - %s' % six.text_type(exc))
def http_quote(string):
"""
Given a unicode string, will do its dandiest to give you back a
valid ascii charset string you can use in, say, http headers and the
like.
"""
if isinstance(string, six.text_type):
try:
import unidecode
except ImportError:
pass
else:
string = unidecode.unidecode(string)
string = string.encode('ascii', 'replace')
# Wrap in double-quotes for ; , and the like
string = string.replace(b'\\', b'\\\\').replace(b'"', b'\\"')
return '"{0!s}"'.format(string.decode())
def echarts_js_dependencies(context, *args):
dependencies = []
def _add(_x):
if _x not in dependencies:
dependencies.append(_x)
for a in args:
if hasattr(a, 'js_dependencies'):
for d in a.js_dependencies:
_add(d)
elif isinstance(a, six.text_type):
_add(a)
if len(dependencies) > 1:
dependencies.remove('echarts')
dependencies = ['echarts'] + list(dependencies)
links = map(DJANGO_ECHARTS_SETTINGS.host_store.generate_js_link, dependencies)
return template.Template(
'<br/>'.join(['<script src="{link}"></script>'.format(link=l) for l in links])
).render(context)
def _make_hash_value(self, user, timestamp):
return (
six.text_type(user.pk) + six.text_type(timestamp) +
six.text_type(user.profile.email_confirmed)
)
