交叉编译Python-2.7.13到ARM（aarch32）—— 支持sqlite3
Posted
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了交叉编译Python-2.7.13到ARM（aarch32）—— 支持sqlite3相关的知识，希望对你有一定的参考价值。
作者：彭东林
QQ： 405728433
环境
主机： ubuntu14.04 64bit
开发板： qemu + vexpress-a9 （参考： http://www.cnblogs.com/pengdonglin137/p/6442583.html）
工具链： arm-none-linux-gnueabi-gcc (gcc version 4.8.3 20140320)
Python版本： Python-2.7.13
概述
前面一篇博文（交叉编译Python-2.7.13到ARM（aarch32）平台）介绍了移植python到aarch32上面，但是发现有很多模块都不能用，可以在板子上面执行下面的命令测试一下：
1 [[email protected] ]# python /usr/lib/python2.7/test/test___all__.py 2 Traceback (most recent call last): 3 File "/usr/lib/python2.7/test/test___all__.py", line 3, in <module> 4 import unittest 5 File "/usr/lib/python2.7/unittest/__init__.py", line 58, in <module> 6 from .result import TestResult 7 File "/usr/lib/python2.7/unittest/result.py", line 9, in <module> 8 from . import util 9 File "/usr/lib/python2.7/unittest/util.py", line 2, in <module> 10 from collections import namedtuple, OrderedDict 11 File "/usr/lib/python2.7/collections.py", line 20, in <module> 12 from _collections import deque, defaultdict 13 ImportError: No module named _collections
可以看到这里找不到_collections模块。
对比x86_64的编译结果：
1 ls x86_64/build/lib.linux-x86_64-2.7/ 2 array.so* _codecs_hk.so* cPickle.so* _curses_panel.so* future_builtins.so* itertools.so* mmap.so* parser.so* _socket.so* _sysconfigdata.py time.so* 3 audioop.so* _codecs_iso2022.so* crypt.so* _curses.so* grp.so* _json.so* _multibytecodec.so* pyexpat.so* spwd.so* _sysconfigdata.pyc unicodedata.so* 4 binascii.so* _codecs_jp.so* cStringIO.so* datetime.so* _hashlib.so* linuxaudiodev.so* _multiprocessing.so* _random.so* _sqlite3.so* _sysconfigdata.pyo zlib.so* 5 _bisect.so* _codecs_kr.so* _csv.so* _elementtree.so* _heapq.so* _locale.so* nis.so* readline.so* _ssl.so* syslog.so* 6 cmath.so* _codecs_tw.so* _ctypes.so* fcntl.so* _hotshot.so* _lsprof.so* operator.so* resource.so* strop.so* termios.so* 7 _codecs_cn.so* _collections.so* _ctypes_test.so* _functools.so* _io.so* math.so* ossaudiodev.so* select.so* _struct.so* _testcapi.so*
而aarch32的编译结果：
1 $ls aarch32/build/lib.linux2-arm-2.7/ 2 audioop.so* _codecs_iso2022.so* _codecs_tw.so* _ctypes.so* _elementtree.so* _json.so* mmap.so* nis.so* resource.so* termios.so* 3 _codecs_cn.so* _codecs_jp.so* crypt.so* _ctypes_test.so* future_builtins.so* linuxaudiodev.so* _multibytecodec.so* parser.so* _sysconfigdata.py _testcapi.so* 4 _codecs_hk.so* _codecs_kr.so* _csv.so* datetime.so* _hotshot.so* _lsprof.so* _multiprocessing.so* pyexpat.so* _sysconfigdata.pyc
可以看到，aarch32上面缺少了很多库， 比如_collections.so，将来这些库会被安装到/usr/lib/python2.7/lib-dynload下面， 所以下面要说的就是将缺少的这些库弄回来！
正文
1、通过分析setup.py发现问题
在函数build_extensions中刚开始self.extensions中存放的是需要编译库， 通过在加打印：
1 diff --git a/setup.py b/setup.py 2 index 54054c2..bc16bb1 100644 3 --- a/setup.py 4 +++ b/setup.py 5 @@ -178,6 +178,7 @@ class PyBuildExt(build_ext): 6 7 def build_extensions(self): 8 9 + print "build_extensions enter." 10 # Detect which modules should be compiled 11 missing = self.detect_modules() 12 13 @@ -191,6 +192,9 @@ class PyBuildExt(build_ext): 14 extensions.append(ctypes) 15 self.extensions = extensions 16 17 + for ext in self.extensions: 18 + print "extensions: ", ext.name 19 + 20 # Fix up the autodetected modules, prefixing all the source files 21 # with Modules/ and adding Python‘s include directory to the path. 22 (srcdir,) = sysconfig.get_config_vars(‘srcdir‘) 23 @@ -217,6 +221,8 @@ class PyBuildExt(build_ext): 24 # Python header files 25 headers = [sysconfig.get_config_h_filename()] 26 headers += glob(os.path.join(sysconfig.get_path(‘include‘), "*.h")) 27 + 28 + print "builtin_module_names: ", sys.builtin_module_names 29 for ext in self.extensions[:]: 30 ext.sources = [ find_module_file(filename, moddirlist) 31 for filename in ext.sources ] 32 @@ -248,10 +254,15 @@ class PyBuildExt(build_ext): 33 remove_modules.append(line[0]) 34 input.close() 35 36 + print "remove_modules: ", remove_modules 37 + 38 for ext in self.extensions[:]: 39 if ext.name in remove_modules: 40 self.extensions.remove(ext) 41 42 + for ext in self.extensions[:]: 43 + print "extensions: ", ext.name 44 + 45 # When you run "make CC=altcc" or something similar, you really want 46 # those environment variables passed into the setup.py phase. Here‘s 47 # a small set of useful ones. 48 @@ -1618,13 +1629,13 @@ class PyBuildExt(build_ext): 49 50 51 # Platform-specific libraries 52 - if host_platform == ‘linux2‘: 53 + if host_platform == ‘linux2‘ or host_platform == ‘linux2-arm‘: 54 # Linux-specific modules 55 exts.append( Extension(‘linuxaudiodev‘, [‘linuxaudiodev.c‘]) ) 56 else: 57 missing.append(‘linuxaudiodev‘) 58 59 - if (host_platform in (‘linux2‘, ‘freebsd4‘, ‘freebsd5‘, ‘freebsd6‘, 60 + if (host_platform in (‘linux2‘,‘linux2-arm‘ ‘freebsd4‘, ‘freebsd5‘, ‘freebsd6‘, 61 ‘freebsd7‘, ‘freebsd8‘) 62 or host_platform.startswith("gnukfreebsd")): 63 exts.append( Extension(‘ossaudiodev‘, [‘ossaudiodev.c‘]) ) 64 @@ -1755,6 +1766,10 @@ class PyBuildExt(build_ext): 65 ## ext = Extension(‘xx‘, [‘xxmodule.c‘]) 66 ## self.extensions.append(ext) 67 68 +# print "missing: ", missing 69 +# for ext in self.extensions: 70 +# print "extensions: ", ext.name 71 + 72 return missing 73 74 def detect_tkinter_explicitly(self): 75 @@ -2229,6 +2244,8 @@ Topic :: Software Development 76 """ 77 78 def main(): 79 + print "sys.path: ", sys.path 80 + print "cross_compiling: ", cross_compiling 81 # turn off warnings when deprecated modules are imported 82 import warnings 83 warnings.filterwarnings("ignore",category=DeprecationWarning)
然后执行./mk2_make.sh可以看到：
1 sys.path: [‘/home/pengdonglin/src/qemu/python_cross_compile/Python-2.7.13‘, ‘/home/pengdonglin/src/qemu/python_cross_compile/aarch32/build/lib.linux2-arm-2.7‘, ‘/home/pengdonglin/src/qemu/python_cross_compile/Python-2.7.13/Lib‘, ‘/home/pengdonglin/src/qemu/python_cross_compile/Python-2.7.13/Lib/plat-linux2‘, ‘/usr/lib/python2.7‘, ‘/usr/lib/python2.7/plat-x86_64-linux-gnu‘, ‘/usr/lib/python2.7/lib-tk‘, ‘/usr/lib/python2.7/lib-old‘, ‘/usr/lib/python2.7/lib-dynload‘] 2 cross_compiling: True 3 build_extensions enter. 4 extensions: _struct 5 extensions: _ctypes_test 6 extensions: array 7 extensions: cmath 8 extensions: math 9 extensions: strop 10 extensions: time 11 extensions: datetime 12 extensions: itertools 13 extensions: future_builtins 14 extensions: _random 15 extensions: _collections 16 extensions: _bisect 17 extensions: _heapq 18 extensions: operator 19 extensions: _io 20 extensions: _functools 21 extensions: _json 22 extensions: _testcapi 23 extensions: _hotshot 24 extensions: _lsprof 25 extensions: unicodedata 26 extensions: _locale 27 extensions: fcntl 28 extensions: pwd 29 extensions: grp 30 extensions: spwd 31 extensions: select 32 extensions: parser 33 extensions: cStringIO 34 extensions: cPickle 35 extensions: mmap 36 extensions: syslog 37 extensions: audioop 38 extensions: crypt 39 extensions: _csv 40 extensions: _socket 41 extensions: _sha 42 extensions: _md5 43 extensions: _sha256 44 extensions: _sha512 45 extensions: termios 46 extensions: resource 47 extensions: nis 48 extensions: binascii 49 extensions: pyexpat 50 extensions: _elementtree 51 extensions: _multibytecodec 52 extensions: _codecs_kr 53 extensions: _codecs_jp 54 extensions: _codecs_cn 55 extensions: _codecs_tw 56 extensions: _codecs_hk 57 extensions: _codecs_iso2022 58 extensions: _multiprocessing 59 extensions: linuxaudiodev 60 extensions: _ctypes 61 builtin_module_names: (‘__builtin__‘, ‘__main__‘, ‘_ast‘, ‘_bisect‘, ‘_codecs‘, ‘_collections‘, ‘_functools‘, ‘_heapq‘, ‘_io‘, ‘_locale‘, ‘_md5‘, ‘_random‘, ‘_sha‘, ‘_sha256‘, ‘_sha512‘, ‘_socket‘, ‘_sre‘, ‘_struct‘, ‘_symtable‘, ‘_warnings‘, ‘_weakref‘, ‘array‘, ‘binascii‘, ‘cPickle‘, ‘cStringIO‘, ‘cmath‘, ‘errno‘, ‘exceptions‘, ‘fcntl‘, ‘gc‘, ‘grp‘, ‘imp‘, ‘itertools‘, ‘marshal‘, ‘math‘, ‘operator‘, ‘posix‘, ‘pwd‘, ‘select‘, ‘signal‘, ‘spwd‘, ‘strop‘, ‘sys‘, ‘syslog‘, ‘thread‘, ‘time‘, ‘unicodedata‘, ‘xxsubtype‘, ‘zipimport‘, ‘zlib‘) 62 remove_modules: [‘DESTLIB=$(LIBDEST)‘, ‘MACHDESTLIB=$(BINLIBDEST)‘, ‘DESTPATH=‘, ‘SITEPATH=‘, ‘TESTPATH=‘, ‘MACHDEPPATH=:$(PLATDIR)‘, ‘EXTRAMACHDEPPATH=‘, ‘TKPATH=:lib-tk‘, ‘OLDPATH=:lib-old‘, ‘COREPYTHONPATH=$(DESTPATH)$(SITEPATH)$(TESTPATH)$(MACHDEPPATH)$(EXTRAMACHDEPPATH)$(TKPATH)$(OLDPATH)‘, ‘PYTHONPATH=$(COREPYTHONPATH)‘, ‘posix‘, ‘errno‘, ‘pwd‘, ‘_sre‘, ‘_codecs‘, ‘_weakref‘, ‘zipimport‘, ‘_symtable‘, ‘GLHACK=-Dclear=__GLclear‘, ‘xxsubtype‘] 63 extensions: _ctypes_test 64 extensions: datetime 65 extensions: future_builtins 66 extensions: _json 67 extensions: _testcapi 68 extensions: _hotshot 69 extensions: _lsprof 70 extensions: parser 71 extensions: mmap 72 extensions: audioop 73 extensions: crypt 74 extensions: _csv 75 extensions: termios 76 extensions: resource 77 extensions: nis 78 extensions: pyexpat 79 extensions: _elementtree 80 extensions: _multibytecodec 81 extensions: _codecs_kr 82 extensions: _codecs_jp 83 extensions: _codecs_cn 84 extensions: _codecs_tw 85 extensions: _codecs_hk 86 extensions: _codecs_iso2022 87 extensions: _multiprocessing 88 extensions: linuxaudiodev 89 extensions: _ctypes 90 Python build finished, but the necessary bits to build these modules were not found: 91 _bsddb _curses _curses_panel 92 _sqlite3 _ssl _tkinter 93 bsddb185 bz2 dbm 94 dl gdbm imageop 95 ossaudiodev readline sunaudiodev 96 zlib 97 To find the necessary bits, look in setup.py in detect_modules() for the module‘s name.
在刚开始的时候，self.extensions中还是全的，但是经过下面的处理后， 很多库都被remove了：
1 for ext in self.extensions[:]: 2 ext.sources = [ find_module_file(filename, moddirlist) 3 for filename in ext.sources ] 4 if ext.depends is not None: 5 ext.depends = [find_module_file(filename, moddirlist) 6 for filename in ext.depends] 7 else: 8 ext.depends = [] 9 # re-compile extensions if a header file has been changed 10 ext.depends.extend(headers) 11 # platform specific include directories 12 ext.include_dirs.extend(incdirlist) 13 # If a module has already been built statically, 14 # don‘t build it here 15 if ext.name in sys.builtin_module_names: 16 self.extensions.remove(ext)
第15行的注释可以看到，如果sys.builtin_module_names中含有extensions中的库，那么这个库就会从extensions中remove。从目前的分析看，交叉编译的时候，setup.py的import sys导入的应该是PC机上面的环境，导致sys.builtin_module_names的值也是PC上面python运行环境的值（可以在PC的终端下输入python，查看sys.builtin_module_names的值）。
2、 解决
这里为了简单起见，我们只需把刚才出问题的判断注释掉，如下：
1 @@ -233,8 +239,8 @@ class PyBuildExt(build_ext): 2 3 # If a module has already been built statically, 4 # don‘t build it here 5 - if ext.name in sys.builtin_module_names: 6 - self.extensions.remove(ext) 7 + #if ext.name in sys.builtin_module_names: 8 + # self.extensions.remove(ext) 9 10 # Parse Modules/Setup and Modules/Setup.local to figure out which 11 # modules are turned on in the file.
然后重新配置、编译、安装， 最后重新制作ramdisk文件，启动板子，重新执行下面的测试：
1 [[email protected] ]# python /usr/lib/python2.7/test/test___all__.py 2 test_all (__main__.AllTest) ... BaseHTTPServer 3 Bastion 4 CGIHTTPServer 5 ConfigParser 6 Cookie 7 DocXMLRPCServer 8 HTMLParser 9 MimeWriter 10 Queue 11 SimpleHTTPServer 12 ... ... 13 ‘xml.sax.xmlreader‘, ‘xmllib‘, ‘xmlrpclib‘] 14 Following modules failed to be imported: [‘ctypes.wintypes‘, ‘dbhash‘, ‘gzip‘, ‘idlelib.AutoComplete‘] 15 ok 16 ---------------------------------------------------------------------- 17 Ran 1 test in 9.345s 18 OK
可以看到，测试成功了。
下面开始一致sqlite3到板子上面，同时让python也增加多sqlite3的支持。
3、支持sqlite3
首先到http://www.sqlite.org/download.html 下载最新的sqlite3的源码，这里我用的是sqlite-autoconf-3170000.tar.gz，然后进行交叉编译，下面是交叉编译的脚本mk.sh:
1 #!/bin/bash 2 export PATH=/home/pengdonglin/src/qemu/aarch32/arm-2014.05/bin:$PATH 3 4 ../sqlite-autoconf-3170000/configure --host=arm-none-linux-gnueabi 5 --prefix=`pwd` 6 7 make -j4 8 make install
然后修改制作ramdisk的脚本：
1 #!/bin/bash 2 3 sudo rm -rf rootfs 4 sudo rm -rf tmpfs 5 sudo rm -rf ramdisk* 6 7 sudo mkdir rootfs 8 sudo cp ../busybox-1.24.2/_install/* rootfs/ -raf 9 10 sudo mkdir -p rootfs/proc/ 11 sudo mkdir -p rootfs/sys/ 12 sudo mkdir -p rootfs/tmp/ 13 sudo mkdir -p rootfs/root/ 14 sudo mkdir -p rootfs/var/ 15 sudo mkdir -p rootfs/mnt/ 16 17 sudo cp etc rootfs/ -arf 18 19 sudo cp -arf ../arm-2014.05/arm-none-linux-gnueabi/libc/lib rootfs/ 20 21 #python 22 sudo mkdir -p rootfs/usr 23 pushd rootfs/usr 24 sudo cp -raf /home/pengdonglin/qemu/thiry_part/Python/aarch32/bin . 25 sudo cp -raf /home/pengdonglin/qemu/thiry_part/Python/aarch32/lib . 26 sudo cp -raf /home/pengdonglin/qemu/thiry_part/Python/aarch32/include . 27 sudo cp -raf /home/pengdonglin/qemu/thiry_part/Python/aarch32/share . 28 sudo /home/pengdonglin/qemu/aarch32/arm-2014.05/bin/arm-none-linux-gnueabi-strip lib/python* 29 popd 30 31 #sqlite3 32 sudo cp -raf /home/pengdonglin/qemu/thiry_part/SQlite3/aarch32/bin/* rootfs/bin/ 33 sudo cp -raf /home/pengdonglin/qemu/thiry_part/SQlite3/aarch32/include/* rootfs/include/ 34 sudo cp -raf /home/pengdonglin/qemu/thiry_part/SQlite3/aarch32/lib/* rootfs/lib/ 35 sudo cp -raf /home/pengdonglin/qemu/thiry_part/SQlite3/aarch32/share/* rootfs/usr/share 36 37 38 sudo mkdir -p rootfs/dev/ 39 sudo mknod rootfs/dev/tty1 c 4 1 40 sudo mknod rootfs/dev/tty2 c 4 2 41 sudo mknod rootfs/dev/tty3 c 4 3 42 sudo mknod rootfs/dev/tty4 c 4 4 43 sudo mknod rootfs/dev/console c 5 1 44 sudo mknod rootfs/dev/null c 1 3 45 46 sudo rm -rf rootfs/lib/*.a 47 sudo rm -rf rootfs/lib/*.la 48 sudo ../arm-2014.05/bin/arm-none-linux-gnueabi-strip rootfs/lib/* 49 50 sudo dd if=/dev/zero of=ramdisk bs=1M count=100 51 sudo mkfs.ext4 -F ramdisk 52 53 sudo mkdir -p tmpfs 54 sudo mount -t ext4 ramdisk ./tmpfs/ -o loop 55 sudo cp -raf rootfs/* tmpfs/ 56 sudo umount tmpfs 57 58 sudo gzip --best -c ramdisk > ramdisk.gz 59 sudo mkimage -n "ramdisk" -A arm -O linux -T ramdisk -C gzip -d ramdisk.gz ramdisk.img
这样在板子上面就可以使用sqlite3了，但是此时python还无法使用，需要重新编译python，并指定sqlite3的lib和include的路径，修改mk1_config.sh如下：
#!/bin/bash export PATH=/home/pengdonglin/qemu/aarch32/arm-2014.05/bin:$PATH ../Python-2.7.13/configure --prefix=`pwd` --host=arm-none-linux-gnueabi --build=x86_64-linux-gnu --enable-ipv6 --enable-shared ac_cv_file__dev_ptmx="yes" ac_cv_file__dev_ptc="no" LDFLAGS="-L/home/pengdonglin/qemu/thiry_part/SQlite3/aarch32/lib" CPPFLAGS="-I/home/pengdonglin/qemu/thiry_part/SQlite3/aarch32/include"
这样在Makefile调用setup.py时就会将sqlite3相关的模块编译进来，然后再次执行mk2_make.sh和mk3_install.sh，然后我们可以检查一下：
1 $ls aarch32/build/lib.linux2-arm-2.7/ 2 array.so* _codecs_hk.so* cPickle.so* datetime.so* _heapq.so* _locale.so* _multiprocessing.so* _random.so* _socket.so* _sysconfigdata.pyc 3 audioop.so* _codecs_iso2022.so* crypt.so* _elementtree.so* _hotshot.so* _lsprof.so* nis.so* resource.so* spwd.so* syslog.so* 4 binascii.so* _codecs_jp.so* cStringIO.so* fcntl.so* _io.so* math.so* operator.so* select.so* _sqlite3.so* termios.so* 5 _bisect.so* _codecs_kr.so* _csv.so* _functools.so* itertools.so* _md5.so* ossaudiodev.so* _sha256.so* strop.so* _testcapi.so* 6 cmath.so* _codecs_tw.so* _ctypes.so* future_builtins.so* _json.so* mmap.so* parser.so* _sha512.so* _struct.so* time.so* 7 _codecs_cn.so* _collections.so* _ctypes_test.so* grp.so* linuxaudiodev.so* _multibytecodec.so* pyexpat.so* _sha.so* _sysconfigdata.py unicodedata.so*
可以看到，库已经很全了。
4、测试
重新制作ramdisk文件，启动系统。
编写测试sqlite3的脚本sq_demo.py如下：
1 #!/usr/bin/python 2 3 import sqlite3 4 5 #open database 6 conn = sqlite3.connect(‘test.db‘) 7 print "Opened database successfully"; 8 9 conn.execute(‘‘‘CREATE TABLE COMPANY 10 (ID INT PRIMARY KEY NOT NULL, 11 NAME TEXT NOT NULL, 12 AGE INT NOT NULL, 13 ADDRESS CHAR(50), 14 SALARY REAL);‘‘‘) 15 print "Table created successfully"; 16 17 #insert 18 conn.execute("INSERT INTO COMPANY (ID,NAME,AGE,ADDRESS,SALARY) 19 VALUES (1, ‘Paul‘, 32, ‘California‘, 20000.00 )"); 20 21 conn.execute("INSERT INTO COMPANY (ID,NAME,AGE,ADDRESS,SALARY) 22 VALUES (2, ‘Allen‘, 25, ‘Texas‘, 15000.00 )"); 23 24 conn.execute("INSERT INTO COMPANY (ID,NAME,AGE,ADDRESS,SALARY) 25 VALUES (3, ‘Teddy‘, 23, ‘Norway‘, 20000.00 )"); 26 27 conn.execute("INSERT INTO COMPANY (ID,NAME,AGE,ADDRESS,SALARY) 28 VALUES (4, ‘Mark‘, 25, ‘Rich-Mond ‘, 65000.00 )"); 29 30 conn.commit() 31 print "Records created successfully"; 32 33 #select 34 cursor = conn.execute("SELECT id, name, address, salary from COMPANY") 35 for row in cursor: 36 print "ID = ", row[0] 37 print "NAME = ", row[1] 38 print "ADDRESS = ", row[2] 39 print "SALARY = ", row[3], "\\n" 40 41 print "Operation done successfully"; 42 43 #delect 44 conn.execute("DELETE from COMPANY where ID=2;") 45 conn.commit() 46 print "Total number of rows deleted :", conn.total_changes 47 48 cursor = conn.execute("SELECT id, name, address, salary from COMPANY") 49 for row in cursor: 50 print "ID = ", row[0] 51 print "NAME = ", row[1] 52 print "ADDRESS = ", row[2] 53 print "SALARY = ", row[3], "\\n" 54 55 print "Operation done successfully"; 56 57 conn.close()
下面是输出结果：
[[email protected] ]# python /tmp/sq_demo.py Opened database successfully Table created successfully Records created successfully ID = 1 NAME = Paul ADDRESS = California SALARY = 20000.0 ID = 2 NAME = Allen ADDRESS = Texas SALARY = 15000.0 ID = 3 NAME = Teddy ADDRESS = Norway SALARY = 20000.0 ID = 4 NAME = Mark ADDRESS = Rich-Mond SALARY = 65000.0 Operation done successfully Total number of rows deleted : 5 ID = 1 NAME = Paul ADDRESS = California SALARY = 20000.0 ID = 3 NAME = Teddy ADDRESS = Norway SALARY = 20000.0 ID = 4 NAME = Mark ADDRESS = Rich-Mond SALARY = 65000.0 Operation done successfully
完。