Python + Twisted + FtpClient + SOCKS
Posted
技术标签:
【中文标题】Python + Twisted + FtpClient + SOCKS【英文标题】： 【发布时间】：2012-12-26 12:49:35 【问题描述】：我刚开始使用 Twisted。我想连接到 FTP 服务器并执行一些基本操作（如果可能，使用线程）。我正在使用这个example。
哪个做得很好。问题是如何在代码中添加 SOCKS4/5 代理使用？有人可以提供一个工作示例吗？我也试过这个link。
但是，
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
An example of using the FTP client
"""
# Twisted imports
from twisted.protocols.ftp import FTPClient, FTPFileListProtocol
from twisted.internet.protocol import Protocol, ClientCreator
from twisted.python import usage
from twisted.internet import reactor, endpoints
# Socks support test
from socksclient import SOCKSv4ClientProtocol, SOCKSWrapper
from twisted.web import client
# Standard library imports
import string
import sys
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
class BufferingProtocol(Protocol):
"""Simple utility class that holds all data written to it in a buffer."""
def __init__(self):
self.buffer = StringIO()
def dataReceived(self, data):
self.buffer.write(data)
# Define some callbacks
def success(response):
print 'Success! Got response:'
print '---'
if response is None:
print None
else:
print string.join(response, '\n')
print '---'
def fail(error):
print 'Failed. Error was:'
print error
def showFiles(result, fileListProtocol):
print 'Processed file listing:'
for file in fileListProtocol.files:
print ' %s: %d bytes, %s' \
% (file['filename'], file['size'], file['date'])
print 'Total: %d files' % (len(fileListProtocol.files))
def showBuffer(result, bufferProtocol):
print 'Got data:'
print bufferProtocol.buffer.getvalue()
class Options(usage.Options):
optParameters = [['host', 'h', 'example.com'],
['port', 'p', 21],
['username', 'u', 'webmaster'],
['password', None, 'justapass'],
['passive', None, 0],
['debug', 'd', 1],
]
# Socks support
def wrappercb(proxy):
print "connected to proxy", proxy
pass
def run():
def sockswrapper(proxy, url):
dest = client._parse(url) # scheme, host, port, path
endpoint = endpoints.TCP4ClientEndpoint(reactor, dest[1], dest[2])
return SOCKSWrapper(reactor, proxy[1], proxy[2], endpoint)
# Get config
config = Options()
config.parseOptions()
config.opts['port'] = int(config.opts['port'])
config.opts['passive'] = int(config.opts['passive'])
config.opts['debug'] = int(config.opts['debug'])
# Create the client
FTPClient.debug = config.opts['debug']
creator = ClientCreator(reactor, FTPClient, config.opts['username'],
config.opts['password'], passive=config.opts['passive'])
#creator.connectTCP(config.opts['host'], config.opts['port']).addCallback(connectionMade).addErrback(connectionFailed)
# Socks support
proxy = (None, '1.1.1.1', 1111, True, None, None)
sw = sockswrapper(proxy, "ftp://example.com")
d = sw.connect(creator)
d.addCallback(wrappercb)
reactor.run()
def connectionFailed(f):
print "Connection Failed:", f
reactor.stop()
def connectionMade(ftpClient):
# Get the current working directory
ftpClient.pwd().addCallbacks(success, fail)
# Get a detailed listing of the current directory
fileList = FTPFileListProtocol()
d = ftpClient.list('.', fileList)
d.addCallbacks(showFiles, fail, callbackArgs=(fileList,))
# Change to the parent directory
ftpClient.cdup().addCallbacks(success, fail)
# Create a buffer
proto = BufferingProtocol()
# Get short listing of current directory, and quit when done
d = ftpClient.nlst('.', proto)
d.addCallbacks(showBuffer, fail, callbackArgs=(proto,))
d.addCallback(lambda result: reactor.stop())
# this only runs if the module was *not* imported
if __name__ == '__main__':
run()
我知道代码是错误的。我需要解决方案。
【问题讨论】：
FTP 是一个复杂的（阅读：不好的）协议，在继续之前需要澄清一些要点。第一：您需要支持主动转账还是 PASV 转账就足够了？第二：一定要用twisted吗？ Python 附带一个 ftplib，可以扩展它以支持 socks。第三：你的 socks 代理是在 VIP 后面还是直接连接到 Internet 的单个主机？ 1.任何类型的转移都可以。 2.它需要具有不同 SOCKS 的多线程。欢迎对如何使用 ftplib 和不同的袜子进行连接的任何想法。 3.只是一个普通的代理。 【参考方案1】：好的，这里有一个解决方案（gist），它使用了python内置的ftplib，以及开源的SocksiPy module。
它不使用twisted，也不显式使用线程，但是使用threading.Thread 和threading.Queue in python's standard threading module 中的threading.Queue 很容易在线程之间进行通信
基本上，我们需要继承 ftplib.FTP 以支持替换我们自己的 create_connection 方法并添加代理配置语义。
“主”逻辑只是配置一个通过 localhost socks 代理连接的 FTP 客户端，例如由ssh -D localhost:1080 socksproxy.example.com 创建的代理，并将 GNU autoconf 的源快照下载到本地磁盘。
import ftplib
import socket
import socks # socksipy (https://github.com/mikedougherty/SocksiPy)
class FTP(ftplib.FTP):
def __init__(self, host='', user='', passwd='', acct='',
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
proxyconfig=None):
"""Like ftplib.FTP constructor, but with an added `proxyconfig` kwarg
`proxyconfig` should be a dictionary that may contain the following
keys:
proxytype - The type of the proxy to be used. Three types
are supported: PROXY_TYPE_SOCKS4 (including socks4a),
PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP
addr - The address of the server (IP or DNS).
port - The port of the server. Defaults to 1080 for SOCKS
servers and 8080 for HTTP proxy servers.
rdns - Should DNS queries be preformed on the remote side
(rather than the local side). The default is True.
Note: This has no effect with SOCKS4 servers.
username - Username to authenticate with to the server.
The default is no authentication.
password - Password to authenticate with to the server.
Only relevant when username is also provided.
"""
self.proxyconfig = proxyconfig or
ftplib.FTP.__init__(self, host, user, passwd, acct, timeout)
def connect(self, host='', port=0, timeout=-999):
'''Connect to host. Arguments are:
- host: hostname to connect to (string, default previous host)
- port: port to connect to (integer, default previous port)
'''
if host != '':
self.host = host
if port > 0:
self.port = port
if timeout != -999:
self.timeout = timeout
self.sock = self.create_connection(self.host, self.port)
self.af = self.sock.family
self.file = self.sock.makefile('rb')
self.welcome = self.getresp()
return self.welcome
def create_connection(self, host=None, port=None):
host, port = host or self.host, port or self.port
if self.proxyconfig:
phost, pport = self.proxyconfig['addr'], self.proxyconfig['port']
err = None
for res in socket.getaddrinfo(phost, pport, 0, socket.SOCK_STREAM):
af, socktype, proto, canonname, sa = res
sock = None
try:
sock = socks.socksocket(af, socktype, proto)
sock.setproxy(**self.proxyconfig)
if self.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
sock.settimeout(self.timeout)
sock.connect((host, port))
return sock
except socket.error as _:
err = _
if sock is not None:
sock.close()
if err is not None:
raise err
else:
raise socket.error("getaddrinfo returns an empty list")
else:
sock = socket.create_connection((host, port), self.timeout)
return sock
def ntransfercmd(self, cmd, rest=None):
size = None
if self.passiveserver:
host, port = self.makepasv()
conn = self.create_connection(host, port)
try:
if rest is not None:
self.sendcmd("REST %s" % rest)
resp = self.sendcmd(cmd)
# Some servers apparently send a 200 reply to
# a LIST or STOR command, before the 150 reply
# (and way before the 226 reply). This seems to
# be in violation of the protocol (which only allows
# 1xx or error messages for LIST), so we just discard
# this response.
if resp[0] == '2':
resp = self.getresp()
if resp[0] != '1':
raise ftplib.error_reply, resp
except:
conn.close()
raise
else:
raise Exception("Active transfers not supported")
if resp[:3] == '150':
# this is conditional in case we received a 125
size = ftplib.parse150(resp)
return conn, size
if __name__ == '__main__':
ftp = FTP(host='ftp.gnu.org', user='anonymous', passwd='guest',
proxyconfig=dict(proxytype=socks.PROXY_TYPE_SOCKS5, rdns=False,
addr='localhost', port=1080))
with open('autoconf-2.69.tar.xz', mode='w') as f:
ftp.retrbinary("RETR /gnu/autoconf/autoconf-2.69.tar.xz", f.write)
详细说明我为什么问我最初的一些问题：
1) 您需要支持主动转账还是 PASV 转账就足够了？
通过 socks 代理进行主动传输要困难得多，因为它们需要使用 PORT 命令。使用 PORT 命令，您的 ftp 客户端告诉 FTP 服务器连接到特定端口（例如，在您的 PC 上）上的 you 以发送数据。这可能不适用于防火墙或 NAT/路由器后面的用户。如果您的 SOCKS 代理服务器没有在防火墙后面，或者具有公共 IP，则可以支持主动传输，但这很复杂：它需要您的 SOCKS 服务器（ssh -D 确实支持）和客户端库（socksipy 不支持）支持远程端口绑定。它还需要在应用程序中使用适当的挂钩（如果passiveserver = False，我的示例会引发异常）来执行远程 BIND 而不是本地 BIND。
2) 一定要用twisted吗？
Twisted 很棒，但我不是最擅长的，而且我还没有找到真正出色的 SOCKS 客户端实现。理想情况下，会有一个库允许您定义和/或将代理链接在一起，返回一个实现IReactorTCP 接口的对象，但我还没有找到类似的东西。
3) 您的 socks 代理是在 VIP 后面还是直接连接到 Internet 的单个主机？
这很重要，因为 PASV 传输安全的工作方式。在 PASV 传输中，客户端要求服务器提供连接端口以开始数据传输。当服务器接受该端口上的连接时，它应该验证客户端与请求传输的连接来自相同的源 IP。如果您的 SOCKS 服务器位于 VIP 后面，则为 PASV 传输建立的连接的出站 IP 不太可能与主要通信连接的出站 IP 匹配。
【讨论】：
以上是关于Python + Twisted + FtpClient + SOCKS的主要内容，如果未能解决你的问题，请参考以下文章