def hdfs_usage(request, root=None):
host = settings.HDFS_STROAGE['hosts']
if not root:
root = settings.HDFS_STROAGE['HDFS_ROOT']
log.info('http://%s/%s', host, root)
client = hdfs.InsecureClient('http://%s' % host)
tree = []
try:
for appcode in client.list(root):
url = 'http://%s/webhdfs/v1%s/%s?op=GETCONTENTSUMMARY' % (host, root, appcode)
data = requests.get(url).json()
if data['ContentSummary']['directoryCount']:
tree.append({
'name': appcode,
'value': round(data['ContentSummary']['length'] / 1000000.0, 2)
})
except HdfsError, e:
log.warn("hdfs_usage error: %s", e)
return HttpResponseNotFound(dumps(tree), content_type='application/json')
return HttpResponse(dumps(tree), content_type='application/json')
python类HttpResponseNotFound()的实例源码
def cancel_order(request):
oid = request.POST.get('id', False)
if not oid:
return HttpResponseNotFound(NOTFOUNDMESSAGE)
try:
obj = Order.objects.get(pk=int(oid), user=request.user)
if obj.is_accept:
return JsonResponse({
'status': 0
})
else:
obj.delete()
return JsonResponse({
'status': 1
})
except:
return JsonResponse({
'status': 0
})
def accept_or_deny(request):
if request.user.usertype:
return HttpResponseForbidden()
else:
op = request.GET.get('op')
try:
opk = int(request.GET.get('opk'))
b = request.user.business
o = Order.objects.get(pk=opk, food__business=b)
except:
return HttpResponseNotFound(NOTFOUNDMESSAGE)
if op == 'accept':
o.is_accept = True
o.save()
elif op == 'deny':
o.delete()
return HttpResponseRedirect(reverse('businessucenterindex'))
def set_info(request, tracker_id):
"""Set browser infos.
:param request:
:param tracker_id:
:return:
"""
if not request.POST.get('infos'):
return HttpResponseBadRequest()
tracker = TrackerInfos.objects.filter(
target_tracker_id=tracker_id, raw=None).order_by('created_at').last()
if not tracker:
return HttpResponseNotFound()
tracker.raw = request.POST['infos']
tracker.save()
return HttpResponse('')
def wfmodule_input(request, pk, format=None):
if request.method == 'GET':
try:
wf_module = WfModule.objects.get(pk=pk)
except WfModule.DoesNotExist:
return HttpResponseNotFound()
if not wf_module.user_authorized_read(request.user):
return HttpResponseForbidden()
# return empty table if this is the first module in the stack
prev_modules = WfModule.objects.filter(workflow=wf_module.workflow, order__lt=wf_module.order)
if not prev_modules:
return HttpResponse(make_render_json(pd.DataFrame()), content_type="application/json")
else:
return table_result(request, prev_modules.last())
# Public access to wfmodule output. Basically just /render with different auth and output format
# NOTE: does not support startrow/endrow at the moment
def wfmodule_public_output(request, pk, type, format=None):
try:
wf_module = WfModule.objects.get(pk=pk)
except WfModule.DoesNotExist:
return HttpResponseNotFound()
if not wf_module.user_authorized_read(request.user):
return HttpResponseNotFound()
table = execute_wfmodule(wf_module)
if type=='json':
d = table.to_json(orient='records')
return HttpResponse(d, content_type="application/json")
elif type=='csv':
d = table.to_csv(index=False)
return HttpResponse(d, content_type="text/csv")
else:
return HttpResponseNotFound()
# Get list of data versions, or set current data version
def wfmodule_dataversion(request, pk, format=None):
try:
wf_module = WfModule.objects.get(pk=pk)
except WfModule.DoesNotExist:
return HttpResponseNotFound()
if request.method == 'GET':
if not wf_module.user_authorized_read(request.user):
return HttpResponseNotFound()
versions = wf_module.list_fetched_data_versions()
current_version = wf_module.get_fetched_data_version()
response = {'versions': versions, 'selected': current_version}
return Response(response)
elif request.method == 'PATCH':
if not wf_module.user_authorized_write(request.user):
return HttpResponseForbidden()
ChangeDataVersionCommand.create(wf_module, datetime.datetime.strptime(request.data['selected'], "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=pytz.UTC))
return Response(status=status.HTTP_204_NO_CONTENT)
def parameterval_png(request, pk):
try:
param = ParameterVal.objects.get(pk=pk)
except ParameterVal.DoesNotExist:
return HttpResponseNotFound()
if not param.wf_module.public_authorized():
return HttpResponseForbidden()
# is this actually in image? totes hardcoded for now
if param.parameter_spec.id_name != 'chart':
return HttpResponseBadRequest()
# decode the base64 payload of the data URI into a png
image_data = param.value.partition('base64,')[2]
binary = base64.b64decode(image_data)
return HttpResponse(binary, content_type='image/png')
def page_not_found(request, template_name='404.html'):
context = {
'request_path': request.path,
'error': {
'title': _('Page not found'),
'message': _("We tried but couldn't find this page, sorry."),
},
}
try:
template = loader.get_template(template_name)
body = template.render(context, request)
content_type = None
except TemplateDoesNotExist:
template = Engine().from_string(
'<h1>Not Found</h1>'
'<p>The requested URL {{ request_path }} was not found on this server.</p>')
body = template.render(Context(context))
content_type = 'text/html'
return HttpResponseNotFound(body, content_type=content_type)
def get_media_file_response(metadata):
if metadata.data_file:
file_path = metadata.data_file.name
filename, extension = os.path.splitext(file_path.split('/')[-1])
extension = extension.strip('.')
dfs = get_storage_class()()
if dfs.exists(file_path):
response = response_with_mimetype_and_name(
metadata.data_file_type,
filename, extension=extension, show_date=False,
file_path=file_path, full_mime=True)
return response
else:
return HttpResponseNotFound()
else:
return HttpResponseRedirect(metadata.data_value)
def page_not_found(request, template_name='404.html'):
context = {
'request_path': request.path,
'error': {
'title': _('Page not found'),
'message': _("We tried but couldn't find this page, sorry."),
},
}
try:
template = loader.get_template(template_name)
body = template.render(context, request)
content_type = None
except TemplateDoesNotExist:
template = Engine().from_string(
'<h1>Not Found</h1>'
'<p>The requested URL {{ request_path }} was not found on this server.</p>')
body = template.render(Context(context))
content_type = 'text/html'
return HttpResponseNotFound(body, content_type=content_type)
def sources(request):
"""
Content Source backend endpoint.
Uses OpenEdx Course API v.1.0
:param request:
:param course_id:
:return: (JSON) blocks
"""
course_id = request.POST.get('course_id')
if not course_id:
return HttpResponseBadRequest(reason={"error": "`course_id` is a mandatory parameter."})
log.debug('course_ID{}'.format(course_id))
try:
sources_list = get_available_blocks(course_id)
except ObjectDoesNotExist as exc:
return HttpResponseBadRequest(reason={"error": exc.message})
except HttpClientError as exc:
return HttpResponseNotFound(reason={"error": exc.message})
return JsonResponse(data=sources_list, safe=False)
def give_away_weekly(request, **kwargs):
if request.method != 'POST':
return HttpResponseNotFound()
scheduledChore = ScheduledChore.objects.get(pk=kwargs['sc_id'])
if request.user != scheduledChore.person:
return HttpResponseNotFound()
people = Person.objects.all().order_by('-weekly_modifier')
oldPerson = scheduledChore.person
# make sure we don't give it back to the same person...
if people[0] == oldPerson:
newPerson = people[1]
else:
newPerson = people[0]
oldPerson.weekly_modifier += 1
oldPerson.save()
scheduledChore.person = newPerson
scheduledChore.save()
return HttpResponseRedirect('/')
def give_away_daily(request, **kwargs):
if request.method != 'POST':
return HttpResponseNotFound()
scheduledChore = ScheduledChore.objects.get(pk=kwargs['sc_id'])
if request.user != scheduledChore.person:
return HttpResponseNotFound()
people = Person.objects.all().order_by('-daily_modifier')
oldPerson = scheduledChore.person
# make sure we don't give it back to the same person...
if people[0] == oldPerson:
newPerson = people[1]
else:
newPerson = people[0]
oldPerson.weekly_modifier += 1
oldPerson.save()
scheduledChore.person = newPerson
scheduledChore.save()
return HttpResponseRedirect('/')
def page_not_found(request, template_name='404.html'):
context = {
'request_path': request.path,
'error': {
'title': _('Page not found'),
'message': _("We tried but couldn't find this page, sorry."),
},
}
try:
template = loader.get_template(template_name)
body = template.render(context, request)
content_type = None
except TemplateDoesNotExist:
template = Engine().from_string(
'<h1>Not Found</h1>'
'<p>The requested URL {{ request_path }} was not found on this server.</p>')
body = template.render(Context(context))
content_type = 'text/html'
return HttpResponseNotFound(body, content_type=content_type)
views.py 文件源码
项目:Distributed-Multi-User-Scrapy-System-with-a-Web-UI
作者: aaldaber
项目源码
文件源码
阅读 18
收藏 0
点赞 0
评论 0
def delete_project(request, projectname):
try:
project = Project.objects.get(user=request.user, project_name=projectname)
except Project.DoesNotExist:
return HttpResponseNotFound('Nothing is here.')
if request.method == 'GET':
form = DeleteProject()
return render(request, 'deleteproject.html', {'username': request.user.username, 'form': form, 'projectname': projectname})
if request.method == 'POST':
if 'cancel' in request.POST:
return HttpResponseRedirect(reverse("mainpage"))
elif 'submit' in request.POST:
project.delete()
return HttpResponseRedirect(reverse("mainpage"))
else:
return HttpResponseNotFound('Nothing is here.')
views.py 文件源码
项目:Distributed-Multi-User-Scrapy-System-with-a-Web-UI
作者: aaldaber
项目源码
文件源码
阅读 25
收藏 0
点赞 0
评论 0
def itemslist(request, projectname):
try:
project = Project.objects.get(user=request.user, project_name=projectname)
except Project.DoesNotExist:
return HttpResponseNotFound('Nothing is here.')
itemtracker = 0
items = Item.objects.filter(project=project)
itemdata = []
for item in items:
itemdata.append([])
itemdata[itemtracker].append(item.item_name)
fields = Field.objects.filter(item=item)
if fields:
itemdata[itemtracker].append([])
for field in fields:
itemdata[itemtracker][1].append(field.field_name)
itemtracker += 1
return render(request, 'itemslist.html',
{'username': request.user.username, 'project': project.project_name, 'items': itemdata})
views.py 文件源码
项目:Distributed-Multi-User-Scrapy-System-with-a-Web-UI
作者: aaldaber
项目源码
文件源码
阅读 22
收藏 0
点赞 0
评论 0
def deleteitem(request, projectname, itemname):
try:
project = Project.objects.get(user=request.user, project_name=projectname)
except Project.DoesNotExist:
return HttpResponseNotFound('Nothing is here.')
try:
item = Item.objects.get(project=project, item_name=itemname)
except Item.DoesNotExist:
return HttpResponseNotFound('Nothing is here.')
if request.method == 'GET':
# using the form that was used for deleting the project
form = DeleteProject()
return render(request, 'deleteitem.html',
{'username': request.user.username, 'form': form, 'projectname': projectname, 'itemname': itemname})
elif request.method == 'POST':
if 'cancel' in request.POST:
return HttpResponseRedirect(reverse("listitems", args=(projectname,)))
elif 'submit' in request.POST:
item.delete()
return HttpResponseRedirect(reverse("listitems", args=(projectname,)))
views.py 文件源码
项目:Distributed-Multi-User-Scrapy-System-with-a-Web-UI
作者: aaldaber
项目源码
文件源码
阅读 25
收藏 0
点赞 0
评论 0
def pipelinelist(request, projectname):
try:
project = Project.objects.get(user=request.user, project_name=projectname)
except Project.DoesNotExist:
return HttpResponseNotFound('Nothing is here.')
itemtracker = 0
pipelines = Pipeline.objects.filter(project=project)
pipelinedata = []
for pipeline in pipelines:
pipelinedata.append([])
pipelinedata[itemtracker].append(pipeline.pipeline_name)
pipelinedata[itemtracker].append(pipeline.pipeline_order)
itemtracker += 1
return render(request, 'pipelinelist.html', {'username': request.user.username, 'project': project.project_name, 'items': pipelinedata})
views.py 文件源码
项目:Distributed-Multi-User-Scrapy-System-with-a-Web-UI
作者: aaldaber
项目源码
文件源码
阅读 23
收藏 0
点赞 0
评论 0
def deletepipeline(request, projectname, pipelinename):
try:
project = Project.objects.get(user=request.user, project_name=projectname)
except Project.DoesNotExist:
return HttpResponseNotFound('Nothing is here.')
try:
pipeline = Pipeline.objects.get(project=project, pipeline_name=pipelinename)
except Pipeline.DoesNotExist:
return HttpResponseNotFound('Nothing is here.')
if request.method == 'GET':
form = DeleteProject()
return render(request, 'deletepipeline.html',
{'username': request.user.username,
'form': form, 'projectname': project.project_name, 'pipelinename': pipeline.pipeline_name})
elif request.method == 'POST':
if 'cancel' in request.POST:
return HttpResponseRedirect(reverse("listpipelines", args=(project.project_name,)))
elif 'submit' in request.POST:
pipeline.delete()
return HttpResponseRedirect(reverse("listpipelines", args=(project.project_name,)))
views.py 文件源码
项目:Distributed-Multi-User-Scrapy-System-with-a-Web-UI
作者: aaldaber
项目源码
文件源码
阅读 20
收藏 0
点赞 0
评论 0
def scraper(request, projectname):
try:
project = Project.objects.get(user=request.user, project_name=projectname)
except Project.DoesNotExist:
return HttpResponseNotFound('Nothing is here.')
spiderclassnamelabel = "class " + request.user.username.title() + project.project_name.title() + "Spider:"
if request.method == 'GET':
form = Scraper(initial={'function': project.scraper_function})
form.fields['function'].label = spiderclassnamelabel
return render(request, 'addscraper.html', {'username': request.user.username, 'form': form, 'project': project.project_name})
elif request.method == 'POST':
if 'cancel' in request.POST:
return HttpResponseRedirect(reverse("manageproject", args=(projectname,)))
if 'submit' in request.POST:
form = Scraper(request.POST)
form.fields['function'].label = spiderclassnamelabel
if form.is_valid():
project.scraper_function = form.cleaned_data['function']
project.save()
return HttpResponseRedirect(reverse("manageproject", args=(projectname,)))
else:
return render(request, 'addscraper.html',
{'username': request.user.username, 'form': form, 'project': project.project_name})
views.py 文件源码
项目:Distributed-Multi-User-Scrapy-System-with-a-Web-UI
作者: aaldaber
项目源码
文件源码
阅读 19
收藏 0
点赞 0
评论 0
def start_project(request, projectname, worker):
uniqueprojectname = request.user.username + '_' + projectname
try:
project = Project.objects.get(user=request.user, project_name=projectname)
except Project.DoesNotExist:
return HttpResponseNotFound('Nothing is here.')
if request.method == 'POST':
if 'linkgenerator' in worker:
linkgenaddress = settings.LINK_GENERATOR
try:
r = requests.post('%s/schedule.json' % linkgenaddress, data={'project': uniqueprojectname, 'spider': uniqueprojectname}, timeout=(3, None))
except:
pass
elif 'worker' in worker:
workernumber = ''.join(x for x in worker if x.isdigit())
workernumber = int(workernumber)
workeraddress = settings.SCRAPERS[workernumber - 1]
try:
r = requests.post('%s/schedule.json' % workeraddress, data={'project': uniqueprojectname, 'spider': uniqueprojectname}, timeout=(3, None))
except:
pass
return HttpResponse('sent start signal')
views.py 文件源码
项目:Distributed-Multi-User-Scrapy-System-with-a-Web-UI
作者: aaldaber
项目源码
文件源码
阅读 19
收藏 0
点赞 0
评论 0
def start_project_on_all(request, projectname):
uniqueprojectname = request.user.username + '_' + projectname
try:
project = Project.objects.get(user=request.user, project_name=projectname)
except Project.DoesNotExist:
return HttpResponseNotFound('Nothing is here.')
workers = []
workers.append(settings.LINK_GENERATOR)
for worker in settings.SCRAPERS:
workers.append(worker)
if request.method == 'POST':
for worker in workers:
try:
r = requests.post('%s/schedule.json' % worker, data={'project': uniqueprojectname, 'spider': uniqueprojectname}, timeout=(3, None))
except:
pass
return HttpResponse('sent start signal')
def require_config(config_model):
"""View decorator that enables/disables a view based on configuration.
Arguments:
config_model (ConfigurationModel subclass): The class of the configuration
model to check.
Returns:
HttpResponse: 404 if the configuration model is disabled,
otherwise returns the response from the decorated view.
"""
def _decorator(func):
@wraps(func)
def _inner(*args, **kwargs):
if not config_model.current().enabled:
return HttpResponseNotFound()
else:
return func(*args, **kwargs)
return _inner
return _decorator
def get_media_file_response(metadata):
if metadata.data_file:
file_path = metadata.data_file.name
filename, extension = os.path.splitext(file_path.split('/')[-1])
extension = extension.strip('.')
dfs = get_storage_class()()
if dfs.exists(file_path):
response = response_with_mimetype_and_name(
metadata.data_file_type,
filename, extension=extension, show_date=False,
file_path=file_path, full_mime=True)
return response
else:
return HttpResponseNotFound()
else:
return HttpResponseRedirect(metadata.data_value)
def get(self, request):
if not request.user.is_authenticated():
result = unauthorized_response()
else:
# FIXME: not sure what best default behavior is
result = HttpResponseNotFound()
return_url = lti_launch_return_url(request.user)
if return_url:
parsed = urlparse(return_url)
launch_q = list(parse_qs(parsed.query).items())
return_q = list(request.GET.items())
new_q = urlencode(launch_q + return_q, doseq=True)
url = urlunparse(
(parsed[0],
parsed[1],
parsed[2],
parsed[3],
new_q,
parsed[5]))
result = HttpResponseRedirect(url, status=303)
return result
def view_selected_detail(request, query_id):
try:
anno_query = AnnoQuery.objects.get(query_image_id=query_id)
except AnnoQuery.DoesNotExist:
return HttpResponseNotFound("Resource not available")
results = []
for image_id in anno_query.target_image_ids:
(width, height) = get_image_meta(image_id)
results.append({
'id': image_id,
'width': width,
'height': height,
})
return render(request, 'annotator/view_select_detail.html', {
'query': query_id,
'results': results,
})
def page_not_found(request, template_name='404.html'):
"""
Default 404 handler.
Templates: :template:`404.html`
Context:
request_path
The path of the requested URL (e.g., '/app/pages/bad_page/')
"""
context = {'request_path': request.path}
try:
template = loader.get_template(template_name)
body = template.render(context, request)
content_type = None # Django will use DEFAULT_CONTENT_TYPE
except TemplateDoesNotExist:
template = Engine().from_string(
'<h1>Not Found</h1>'
'<p>The requested URL {{ request_path }} was not found on this server.</p>')
body = template.render(Context(context))
content_type = 'text/html'
return http.HttpResponseNotFound(body, content_type=content_type)
def custom_not_found_page(request):
return HttpResponseNotFound(
render_to_string('404.html', request=request))
def log_kill(request):
""" ??connect?? """
pid = request.GET.get('id', '')
log = Log.objects.filter(pid=pid)
if log:
log = log[0]
try:
os.kill(int(pid), 9)
except OSError:
pass
Log.objects.filter(pid=pid).update(is_finished=1, end_time=datetime.datetime.now())
return render_to_response('jlog/log_offline.html', locals(), context_instance=RequestContext(request))
else:
return HttpResponseNotFound(u'?????!')