Python Proxy isn't handling POST requests

Question

I'm creating a simple Python proxy that logs all requests sent through it. For now, everything works except POST requests (I can't vote on StackOverflow questions).

I haven't defined an explicit POST function, but I am using the GET function instead.

Could anyone tell me where the problem is?

This is my code:

#!/usr/local/bin/python2

import BaseHTTPServer, select, socket, SocketServer, urlparse

class ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
  def __init__(self, request, client_address, server):
      self.request = request
      self.client_address = client_address
      self.server = server
      self.setup()

      self.socket = socket

      self.server_version = 'TestProxy 1.0'
      self.request_buffer_size = 0

      try:
        self.handle()
      finally:
        self.finish()

  def connect_to(self, location, destination_socket):
    i = location.find(':')

    if i >= 0:
      host = location[:i]
      port = int(location[i + 1:])
    else:
      host = location
      port = 80

    print 'Connecting to {0}:{1}'.format(host, port)

    try:
      self.socket.connect((host, port))
    except socket.error, arg:
      try:
        msg = arg[1]
      except:
        msg = arg

      self.send_error(404, msg)

      return 0
    return 1

  def do_CONNECT(self):
    self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    try:
      if self.connect_to(self.path, socket):
        self.log_request(200)
        self.wfile.write(self.protocol_version + ' 200 Connection established\n')
        self.wfile.write('Proxy-agent: {0}\n'.format(self.version_string()))
        self.wfile.write('\n')

        self.read_write(self.socket, 300)
    finally:
      print 'Connection closed.'
      self.socket.close()
      self.connection.close()

  def do_GET(self):
    scm, location, path, parameters, query, fragment = urlparse.urlparse(self.path, 'http')

    if scm != 'http' or fragment or not location:
      self.send_error(400, 'bad url {0}'.format(self.path))
      return

    self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    try:
      if self.connect_to(location, self.socket):
        self.log_request()
        self.socket.send('{0} {1} {2}\n'.format(self.command, urlparse.urlunparse(('', '', path, parameters, query, '')), self.request_version))
        self.headers['Connection'] = 'close'

        del self.headers['Proxy-Connection']

        for key, value in self.headers.items():
          self.socket.send('{0}: {1}\n'.format(key, value))

        self.socket.send('\n')
        self.read_write(self.socket)
    finally:
      print 'Closing connection...'

      self.socket.close()
      self.connection.close()

  def read_write(self, socket, max_idling = 100):
    iw = [self.connection, socket]
    ow = []
    count = 0

    while count != max_idling:
      count += 1
      read_list, write_list, exception_list = select.select(iw, ow, iw, 3)

      if exception_list:
        break

      if read_list:
        for item in read_list:
          out = self.connection if item is socket else socket
          data = item.recv(8192)

          if data:
            out.send(data)
            count = 0
      else:
        pass
        #print 'Idle for {0}'.format(count)

  do_HEAD   = do_GET
  do_POST   = do_GET
  do_PUT    = do_GET
  do_DELETE = do_GET

class ThreadingHTTPServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): pass

if __name__ == '__main__':
  BaseHTTPServer.test(ProxyHandler, ThreadingHTTPServer)

This part handles the POST/GET requests:

  def do_GET(self):
    scm, location, path, parameters, query, fragment = urlparse.urlparse(self.path, 'http')

    if scm != 'http' or fragment or not location:
      self.send_error(400, 'bad url {0}'.format(self.path))
      return

    self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    try:
      if self.connect_to(location, self.socket):
        self.log_request()
        self.socket.send('{0} {1} {2}\n'.format(self.command, urlparse.urlunparse(('', '', path, parameters, query, '')), self.request_version))
        self.headers['Connection'] = 'close'

        del self.headers['Proxy-Connection']

        for key, value in self.headers.items():
          self.socket.send('{0}: {1}\n'.format(key, value))

        self.socket.send('\n')
        self.read_write(self.socket)
    finally:
      print 'Closing connection...'

      self.socket.close()
      self.connection.close()

Answer 1

The read-select on self.connection never returns, and if one manually reads from that object, no data is coming. It looks like you have to read from self.rfile.

Note that calling select on self.rfile doesn't return, but self.rfile.read() does work. That is probably because self.rfile is a Python file-like object, and the server process has already read the start of the POST body from the OS.