mysql+cicd通过tar完成大库部署

Posted 2020-12-12 Meditation

dockerfile:

FROM founder/mysql:5.7

RUN apt-get update && apt-get update ; apt-get install -y parallel && chown -R mysql:mysql /docker-entrypoint-initdb.d

ADD init.sh /docker-entrypoint-initdb.d/
COPY wisekg.tar.gz /docker-entrypoint-initdb.d/
COPY kb.tar.gz /docker-entrypoint-initdb.d/

ADD mysqld.cnf /etc/mysql/mysql.conf.d/

 

init.sh:

#!/bin/bash
whoami
tar zxvf /docker-entrypoint-initdb.d/wisekg.tar.gz -C /docker-entrypoint-initdb.d
mkdir /docker-entrypoint-initdb.d/imdb/split
cd /docker-entrypoint-initdb.d/imdb/split
for i in ../*.txt ; do echo $i ; split -d -a 6 -l 100000 -- $i `basename $i .txt`.; ls|grep `basename $i .txt`|xargs -n1 -i{} mv {} {}.txt; rm -fv $i;  done
for i in `ls *.*|sed ‘s/^[^.]+.//‘|sort -u` ; do
  mkdir ../split-$i
  for j in $i ; do mv $j ../split-$i/$i ; done
done
cd /docker-entrypoint-initdb.d/imdb/
 time ( mysql -uroot -p"$MYSQL_ROOT_PASSWORD"  -e ‘drop database if exists $MYSQL_DATABASE;create database $MYSQL_DATABASE;set global FOREIGN_KEY_CHECKS=0;‘; (echo "SET FOREIGN_KEY_CHECKS=0;";cat ./*.sql) | mysql -uroot -p"$MYSQL_ROOT_PASSWORD" "$MYSQL_DATABASE" ; parallel -j32 ‘mysqlimport --use-threads=32 -uroot -p"$MYSQL_ROOT_PASSWORD" "$MYSQL_DATABASE"  /docker-entrypoint-initdb.d/imdb/{}/*.txt‘ ::: split-* )

whoami
tar zxvf /docker-entrypoint-initdb.d/kb.tar.gz -C /docker-entrypoint-initdb.d
mkdir /docker-entrypoint-initdb.d/imdbkb/split
cd /docker-entrypoint-initdb.d/imdbkb/split
for i in ../*.txt ; do echo $i ; split -d -a 6 -l 100000 -- $i `basename $i .txt`.; ls|grep `basename $i .txt`|xargs -n1 -i{} mv {} {}.txt; rm -fv $i;  done
for i in `ls *.*|sed ‘s/^[^.]+.//‘|sort -u` ; do
  mkdir ../split-$i
  for j in $i ; do mv $j ../split-$i/$i ; done
done
cd /docker-entrypoint-initdb.d/imdbkb/
 time ( mysql -uroot -p"$MYSQL_ROOT_PASSWORD"  -e ‘drop database if exists kb;create database kb;set global FOREIGN_KEY_CHECKS=0;‘; (echo "SET FOREIGN_KEY_CHECKS=0;";cat ./*.sql) | mysql -uroot -p"$MYSQL_ROOT_PASSWORD" kb ; parallel -j32 ‘mysqlimport --use-threads=32 -uroot -p"$MYSQL_ROOT_PASSWORD" kb  /docker-entrypoint-initdb.d/imdbkb/{}/*.txt‘ ::: split-* )

 

mysql.cnf

#Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA

#
# The MySQL  Server configuration file.
#
# For explanations see
# http://dev.mysql.com/doc/mysql/en/server-system-variables.html

[mysqld]
pid-file    = /var/run/mysqld/mysqld.pid
socket        = /var/run/mysqld/mysqld.sock
datadir        = /var/lib/mysql
#log-error    = /var/log/mysql/error.log
# By default we only accept connections from localhost
#bind-address    = 127.0.0.1
# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0


innodb_buffer_pool_size = 4G
innodb_write_io_threads = 64
innodb_read_io_threads = 64

innodb_thread_concurrency=0

innodb_flush_method=O_DIRECT
#innodb_io_capacity=2000
#innodb_io_capacity_max=4000
max_allowed_packet=2G
skip-log-bin
sync_binlog=0
secure_file_priv=‘‘

innodb_flush_log_at_trx_commit = 0
innodb_log_buffer_size = 1G
innodb_log_file_size = 3G

innodb_autoinc_lock_mode=2
innodb_doublewrite=0
innodb_old_blocks_pct=5
skip-innodb_adaptive_hash_index

secure_file_priv =