tokenizer.py 文件源码

def __request(input0):
  client = httpclient.HTTPClient()
  output = None
  try:
    print(colored('tokenising.. ','yellow') + input0)
    req    = httpclient.HTTPRequest(tokeniser_serv,method='POST',body=input0)
    resp   = client.fetch(req)
    output = resp.body

    # Decode bytes
    output = output.decode('utf-8')
    # Parse the JSON response to a hash object
    output = json.loads(output)

  except httpclient.HTTPError as e:
    # HTTP error header
    print(colored('HTTP Error : ' + str(e),'red'))
  except Exception as e:
    # Some unhandled error
    print(colored('ERROR : ' + str(e), 'red'))

  client.close()

  print(colored('{Tokenizer received:}','white'))
  print(colored(output,'white'))

  return output