ES实战在Linux下 CentOS 7离线安装Rally2.7.0

Posted 2022-12-30 顧棟

在Linux下 CentOS7离线安装Rally2.7.0

环境要求

1. Python3.8+ 官网下载地址

2. pip20.3+ 官网下载地址

3. git1.9+ 官网下载地址

4. pbzip2 官网下载地址

5. bzip2 官网下载地址

6. JDK 1.8 Oracle版下载地址

7. CentOS7系统需要升级openssl 官网下载地址

   Python3.8需要升级setupools 官网下载地址

8. 安装编译依赖

   yum install curl-devel expat-devel gettext-devel openssl-devel zlib-devel gcc perl-ExtUtils-MakeMaker;
   

Rally离线安装包官网下载地址，找到2.7.0版本，下载文件esrally-dist-linux-2.7.0.tar.gz

部署模式

• Rally与ES分开部署

• 通过自定义用户esrally安装Python3.8+和Rally2.7.0，其他的在root用户下提前安装。

安装步骤

1. 编译依赖部分

   yum install curl-devel expat-devel gettext-devel openssl-devel zlib-devel gcc perl-ExtUtils-MakeMaker;
   

2. pbzip2部分

   # 新建esrally用户与用户组
   groupadd esrally
   useradd -g esrally esrally
   # 安装pbzip2
   mkdir -p /home/esrally/software/pbzip2
   tar -zxvf pbzip2-1.1.13.tar.gz
   cd pbzip2-1.1.13
   make && make install PREFIX=/home/esrally/software/pbzip2
   

3. openssl 升级部分

   openssl version
   tar -zxvf openssl-1.1.1s.tar.gz
   cd openssl-1.1.1s/
   ./config --prefix=/usr/local/openssl
   make && make install
   mv /usr/bin/openssl /usr/bin/openssl.old
   mv /usr/include/openssl /usr/include/openssl.old
   ln -s /usr/local/openssl/bin/openssl /usr/bin/openssl;
   ln -s /usr/local/openssl/include/openssl /usr/include/openssl
   echo "/usr/local/openssl/lib" >> /etc/ld.so.conf
   ldconfig
   
   

4. Python3.8部分

   
   su - esrally
   mkdir -p software
   mkdir -p python
   exit
   # 为esrally安装python？
   tar -xvf /home/esrally/software/Python-3.8.16.tar.xz -C /home/esrally/software/python/
   chown esrally:esrally -R /home/esrally/software/
   su - esrally
   # 编译 Python-3.8.16
   cd software/python/Python-3.8.16/
   ./configure --prefix=/home/esrally/software/python/python3
   # 安装python3
   mkdir -p /home/esrally/software/python/python3
   make && make install
   # 配置环境变量
   vim ~/.bashrc;
   export PATH=$PATH:/home/esrally/software/python/python3/bin
   source ~/.bashrc;
   python3 --version
   
   # 升级setuptools python3方式
   tar -zxvf setuptools-65.6.3.tar.gz
   cd setuptools-65.6.3/
   python3 setup.py build
   python3 setup.py install
   pip3 show setuptools
   pip3 uninstall setuptools
   pip3 show setuptools
   

5. esrally部分

   # 解压安装包
   tar -xzf esrally-dist-linux-*.tar.gz -C /opt/software/
   # 执行安装脚本
   sudo ./opt/software/esrally-dist-linux-*/install.sh
   # 验证是否安装成功
   esrally --version
   

安装错误小结

错误一，python没有成功编译，在安装编译依赖之后，重新安装编译安装Python3

subprocess.CalledProcessError: Command '['/home/esrally/software/python/Python-3.8.16/python', '-c', '\\nimport runpy\\nimport sys\\nsys.path = [\\'/tmp/tmpoe84zvss/setuptools-56.0.0-py3-none-any.whl\\', \\'/tmp/tmpoe84zvss/pip-22.0.4-py3-none-any.whl\\'] + sys.path\\nsys.argv[1:] = [\\'install\\', \\'--no-cache-dir\\', \\'--no-index\\', \\'--find-links\\', \\'/tmp/tmpoe84zvss\\', \\'--root\\', \\'/\\', \\'--upgrade\\', \\'setuptools\\', \\'pip\\']\\nrunpy.run_module("pip", run_name="__main__", alter_sys=True)\\n']' returned non-zero exit status 1.
make: *** [install] Error 1

错误二：gcc没有成功安装，需要yum安装

[esrally@~ pbzip2-1.1.13]$ make && make install PREFIX=/home/esrally/software/pbzip2
g++ -O2 -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -DUSE_STACKSIZE_CUSTOMIZATION -pthread -D_POSIX_PTHREAD_SEMANTICS  pbzip2.cpp BZ2StreamScanner.cpp ErrorContext.cpp -o pbzip2 -lbz2 -lpthread
make: g++: Command not found
make: *** [pbzip2] Error 127

yum install "gcc-c++.x86_64"

错误三：bzip2c没有成功安装，需要yum安装

[esrally@~ pbzip2-1.1.13]$ make && make install PREFIX=/home/esrally/software/pbzip2
g++ -O2 -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -DUSE_STACKSIZE_CUSTOMIZATION -pthread -D_POSIX_PTHREAD_SEMANTICS  pbzip2.cpp BZ2StreamScanner.cpp ErrorContext.cpp -o pbzip2 -lbz2 -lpthread
pbzip2.cpp:33:19: fatal error: bzlib.h: No such file or directory
 #include <bzlib.h>
                   ^
compilation terminated.
make: *** [pbzip2] Error 1

yum install  bzip2-devel.x86_64

错误四：python中setuptools版本过低，需要单独升级

Processing ./esrally-dist-2.7.0/bin/py-cpuinfo-7.0.0.tar.gz
  Preparing metadata (setup.py) ... error
  error: subprocess-exited-with-error
  
  × python setup.py egg_info did not run successfully.
  │ exit code: 1
  ╰─> [1 lines of output]
      ERROR: Can not execute `setup.py` since setuptools is not available in the build environment.
      [end of output]
  
  note: This error originates from a subprocess, and is likely not a problem with pip.
error: metadata-generation-failed

× Encountered error while generating package metadata.
╰─> See above for output.

note: This is an issue with the package mentioned above, not pip.
hint: See above for details.

tar -zxvf setuptools-65.6.3.tar.gz
 cd setuptools-65.6.3/
python3 setup.py build
python3 setup.py install
pip3 show setuptools
pip3 uninstall setuptools
pip3 show setuptools

错误五：缺少压缩集成包_bz2，这个可以从低版本中找到，然后修改文件名copy至python3.8安装目录lib-dynload下。

[esrally@~ software]$ esrally --version
Traceback (most recent call last):
  File "/home/esrally/software/python/python3/bin/esrally", line 5, in <module>
    from esrally.rally import main
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/esrally/rally.py", line 31, in <module>
    from esrally import (
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/esrally/actor.py", line 25, in <module>
    from esrally import exceptions, log
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/esrally/log.py", line 25, in <module>
    from esrally.utils import io
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/esrally/utils/io.py", line 20, in <module>
    import bz2
  File "/home/esrally/software/python/python3/lib/python3.8/bz2.py", line 19, in <module>
    from _bz2 import BZ2Compressor, BZ2Decompressor
ModuleNotFoundError: No module named '_bz2'

# 移动
mv _bz2.cpython-38-x86_64-linux-gnu.so /home/esrally/software/python/python3/lib/python3.8/lib-dynload/_bz2.cpython-38-x86_64-linux-gnu.so
# 赋权
chmod +x /home/esrally/software/python/python3/lib/python3.8/lib-dynload/_bz2.cpython-38-x86_64-linux-gnu.so

错误六：libbz2.so.1.0文件缺失，通过增加软链接的方式解决

[esrally@~ software]$ esrally --version
Traceback (most recent call last):
  File "/home/esrally/software/python/python3/bin/esrally", line 5, in <module>
    from esrally.rally import main
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/esrally/rally.py", line 31, in <module>
    from esrally import (
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/esrally/actor.py", line 25, in <module>
    from esrally import exceptions, log
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/esrally/log.py", line 25, in <module>
    from esrally.utils import io
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/esrally/utils/io.py", line 20, in <module>
    import bz2
  File "/home/esrally/software/python/python3/lib/python3.8/bz2.py", line 19, in <module>
    from _bz2 import BZ2Compressor, BZ2Decompressor
ImportError: libbz2.so.1.0: cannot open shared object file: No such file or directory

ln -s /usr/lib64/libbz2.so.1.0.6 /usr/lib64/libbz2.so.1.0

错误七：openssl版本过低，需要对系统的openssl进行升级，并重新编译安装Python

[esrally@~ software]$ esrally --version
Traceback (most recent call last):
  File "/home/esrally/software/python/python3/bin/esrally", line 5, in <module>
    from esrally.rally import main
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/esrally/rally.py", line 31, in <module>
    from esrally import (
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/esrally/racecontrol.py", line 26, in <module>
    from esrally import (
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/esrally/mechanic/__init__.py", line 19, in <module>
    from .mechanic import (
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/esrally/mechanic/mechanic.py", line 30, in <module>
    from esrally.mechanic import launcher, provisioner, supplier, team
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/esrally/mechanic/supplier.py", line 28, in <module>
    import docker
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/docker/__init__.py", line 2, in <module>
    from .api import APIClient
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/docker/api/__init__.py", line 2, in <module>
    from .client import APIClient
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/docker/api/client.py", line 10, in <module>
    from .. import auth
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/docker/auth.py", line 7, in <module>
    from .utils import config
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/docker/utils/__init__.py", line 3, in <module>
    from .decorators import check_resource, minimum_version, update_headers
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/docker/utils/decorators.py", line 4, in <module>
    from . import utils
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/docker/utils/utils.py", line 16, in <module>
    from ..tls import TLSConfig
  File "/home/esrally/software/python/python3/lib/python3.8/site-packages/docker/tls.py", line 2, in <module>
    import ssl
  File "/home/esrally/software/python/python3/lib/python3.8/ssl.py", line 98, in <module>
    import _ssl             # if we can't import it, let the error propagate
ModuleNotFoundError: No module named '_ssl'

openssl version
tar -zxvf openssl-1.1.1s.tar.gz
cd openssl-1.1.1s/
./config --prefix=/usr/local/openssl
make && make install
mv /usr/bin/openssl /usr/bin/openssl.old
mv /usr/include/openssl /usr/include/openssl.old
ln -s /usr/local/openssl/bin/openssl /usr/bin/openssl;
ln -s /usr/local/openssl/include/openssl /usr/include/openssl
echo "/usr/local/openssl/lib" >> /etc/ld.so.conf
ldconfig


python3 -c "import sysconfig; print(sysconfig.get_config_var('CONFIG_ARGS'))" '--prefix=/usr/local/python3' '--with-openssl=/usr/local/openssl' '--enable-shared'

# 重编译python

cd software/python/Python-3.8.16/
./configure --prefix=/home/esrally/software/python/python3 --with-openssl=/usr/local/openssl --enable-shared
make && make install

错误八：缺少libssl.so.1.1包，可以在openssl安装目录下寻找，并copy至/usr/lib64文件夹中

openssl: error while loading shared libraries: libssl.so.1.1: cannot open shared object file: No such file or directory

find / -name *libssl.so.1.1

ln -s /usr/local/openssl/lib/libssl.so.1.1 /usr/lib64/libssl.so.1.1
ln -s /usr/local/openssl/lib/libcrypto.so.1.1 /usr/lib64/libcrypto.so.1.1

错误九：缺少libpython3.8.so.1.0包，可以在python目录下寻找，并copy至/usr/lib64文件夹中

find / -name libpython3.8.so.1.0
cp /opt/software/python/Python-3.8.16/libpython3.8.so.1.0  /usr/lib64/

卸载

pip3 install esrally