shell
## 安装脚本
### topic
```bash
#!/usr/bin/env bash
kafka_Home="/mnt/kafka"
bootstrap_server="172.24.61.104:19092"
kafka_group=CLIENT
function delete_topic() {
topic_list=`${kafka_Home}/bin/kafka-topics.sh --list --bootstrap-server $bootstrap_server`
for topic in ${topic_list};
do
{
echo "delete topic [${topic}]"
${kafka_Home}/bin/kafka-topics.sh --bootstrap-server $bootstrap_server --topic ${topic} --delete
} &
done
wait
}
function create_topic() {
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions 1 --topic alarmSimulationData
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions 1 --topic alarmSimulationResult
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions 1 --topic alarmAlgorithmData
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions 1 --topic alarmAlgorithmResult
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions 1 --topic NetReassemble
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions 1 --topic notifyWeb
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions $1 --topic BpmRaw
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions $1 --topic BpmPacket
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions $1 --topic BpmEvent
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions $1 --topic BpmPairing
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions $1 --topic NpmPacket
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions 1 --topic TidmEvent
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions 1 --topic logEvent
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions $1 --topic BpmDelayEvent
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions $1 --topic BpmRealTimeEvent
}
function desc_group() {
if [[ ! -z $1 ]];then
$kafka_Home/bin/kafka-consumer-groups.sh --bootstrap-server $bootstrap_server --describe --group $1
exit 0
fi
$kafka_Home/bin/kafka-consumer-groups.sh --bootstrap-server $bootstrap_server --describe --group $kafka_group
}
function use_help() {
echo "
# 创建topic
bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions 3 --topic weblog
# 修改topic副本
bin/kafka-topics.sh --alter --partitions 4 --topic weblog --bootstrap-server $bootstrap_server
# 删除topic副本
bin/kafka-topics.sh --bootstrap-server $bootstrap_server --topic weblog --delete
# 查看kafka的所有topic
bin/kafka-topics.sh --bootstrap-server $bootstrap_server --list
# 查看单个topic的详细信息
bin/kafka-topics.sh --bootstrap-server $bootstrap_server --topic weblog --describe
Topic:weblog PartitionCount:3 ReplicationFactor:1 Configs:
...
# 查询topic里内容：
bin/kafka-console-consumer.sh --bootstrap-server $bootstrap_server --topic weblog --from-beginning
# 查看消费者consumer group列表
bin/kafka-consumer-groups.sh --bootstrap-server $bootstrap_server --list
# 删除消费组
bin/kafka-consumer-groups.sh --bootstrap-server $bootstrap_server --delete --group consumer-test
# 查看消费者consumer group详情
bin/kafka-consumer-groups.sh --bootstrap-server $bootstrap_server --group consumer-test --describe "
}
case $1 in
help)
use_help
;;
group)
desc_group $2
;;
list)
echo "----- [topic] -----"
${kafka_Home}/bin/kafka-topics.sh --bootstrap-server $bootstrap_server --list
echo "----- [consumer group] -----"
${kafka_Home}/bin/kafka-consumer-groups.sh --bootstrap-server $bootstrap_server --list
;;
delete)
delete_topic
;;
create)
if [[ -z $2 ]];then
echo "Usage: $0 create partitions_count(int) "
exit 0
fi
create_topic $2
;;
*)
echo "----------------------------------------------------------
Usage: $0 help [查看kafka的基本操作命令]
Usage: $0 list [查看topic/消费组]
Usage: $0 group [group_name(可选)] [查看消费组lag]
Usage: $0 delete [删除所有topic]
Usage: $0 create partitions_count(int) [创建topic 分区数]
----------------------------------------------------------"
;;
esac
```
### 移动数据包
```bash
#!/bin/bash
#将数据包移动或拷贝到此
copydir="../$(date +%Y-%m-%d)/in/"
mkdir -p ${copydir}
if [[ -z $1 || -z $2 ]];then
echo "Usage:$0 start_time end_time [null/mv/cp]"
echo "Usage:$0 11:00:50 12:52:55"
echo "Usage:$0" ' "2023-11-09 21:00:50" "2023-11-09 21:52:55"'
echo "Usage:$0 11:00:50 12:52:55 cp # 将数据包cp到${copydir}目录。"
echo "Usage:$0 11:00:50 12:52:55 mv # 将数据包mv到${copydir}目录，不加此参数默认使用mv。 "
exit 1
fi
if [[ -z $3 ]];then
model="mv"
else
case $3 in
'cp')
model="cp"
;;
*)
model="mv"
;;
esac
fi
# 格式化起始和终止时间
start=$(date -d "$1" "+%Y-%m-%d_%H:%M:%S")
stop=$(date -d "$2" "+%Y-%m-%d_%H:%M:%S")
DATE=`date -d "$1" "+%Y-%m-%d"`
# 获取当前路径的所有存包的时间
pcaps=$(ls ./|grep .cap$|grep ${DATE} |awk -F_ '{print $1"_"$2 }')
for pcap in ${pcaps}; do
{
# 获取当前数据包所属日期
pcap_day="$(echo ${pcap}|awk -F_ '{print $1}')"
# 获取当前数据包所属时间
pcap_time="$(echo ${pcap}|awk -F_ '{print $2}'|tr - :)"
# 格式化当前数据包的日期时间
t=$(date -d "$pcap_day $pcap_time" "+%Y-%m-%d_%H:%M:%S")
# 比较数据包时间是否在起始到终止时间内
if [[ "$t" > "$start" && "$t" < "$stop" ]]; then
PCAP_FILE=$(ls | grep .cap|grep ${DATE} |grep $pcap)
echo "$model => ${PCAP_FILE} => ${copydir}"
# 在此处添加拷贝操作的代码
$model ${PCAP_FILE} ${copydir}
fi
} &
done
wait
```
### init
```bash
#!/bin/bash
# 关闭selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
# 关闭防火墙
systemctl stop firewalld
systemctl disable firewalld
# 安装一些常用的软件
yum -y install epel-release openssh-clients net-tools vim tree lrzsz wget bash-completion ntpdate sshpass python3 python3-pip
yum -y install ansible nginx
# yum源更换
function yummakecache(){
mkdir -p /etc/yum.repos.d/backup
mv /etc/yum.repos.d/* /etc/yum.repos.d/backup
wget -O /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
yum repolist
}
# 安装命令补全
yum -y install bash-completion-extra
echo '* - nofile 65535' >>/etc/security/limits.conf
# 修改终端颜色
cat >> ~/.bashrc <<EOF
PS1='[\[\e[34;1m\]\u@\[\e[0m\]\[\e[32;1m\]\H\[\e[0m\]\[\e[31;1m\] \W\[\e[0m\]]# '
EOF
source ~/.bashrc
# 修改ssh服务优化
function sshoptimize(){
sed -ri 's@^#UseDNS yes@UseDNS no@g' /etc/ssh/sshd_config
sed -ri 's#^GSSAPIAuthentication yes#GSSAPIAuthentication no#g' /etc/ssh/sshd_config
grep ^UseDNS /etc/ssh/sshd_config
grep ^GSSAPIAuthentication /etc/ssh/sshd_config
}
function main(){
yummakecache
sshoptimize
}
. ~/.bashrc
# main
```
### rar
```
wget https://www.rarlab.com/rar/rarlinux-x64-3.8.0.tar.gz --no-check-certificate
tar -xf rarlinux-3.8.0.tar.gz
cd rar
make
make install
bug处理，使用的时候出现一个错误
bash: /usr/local/bin/rar: /lib/ld-linux.so.2: bad ELF interpreter: No such file or directory
是因为64位系统中安装了32位程序
解决方法：
yum install glibc.i686
yum install libstdc++.so.6
```
### md5
```
[root@192-168-144-167 swweb]# md5sum sw.zip
c971c474a0957d72ef3cbad94548756a sw.zip
F:\360MoveData\Users\Administrator\Desktop\sw>certutil -hashfile sw.zip md5
MD5 的 sw.zip 哈希:
c971c474a0957d72ef3cbad94548756a
CertUtil: -hashfile 命令成功完成。
```
### kafka
```
yum -y install java
wget https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.8.1/kafka_2.12-2.8.1.tgz --no-check-certificate
tar -zxvf kafka_2.12-2.8.1.tgz
mv kafka_2.12-2.8.1 /usr/bin/
cd /usr/bin/kafka_2.12-2.8.1/config/ && cp server.properties{,.bak}
cat > server.properties.bak <<EOF
broker.id=0
listeners=PLAINTEXT://10.1.128.85:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=localhost:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
EOF
cd /mnt
wget https://dlcdn.apache.org/zookeeper/zookeeper-3.5.10/apache-zookeeper-3.5.10-bin.tar.gz --no-check-certificate
tar -zxf apache-zookeeper-3.5.10-bin.tar.gz
mv apache-zookeeper-3.5.10-bin zookeeper
# 创建数据目录和日志目录
mkdir /mnt/zookeeper/{data,logs}
cat > zoo.cfg <<EOF
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/mnt/zookeeper/data
dataLogDir=/mnt/zookeeper/logs
clientPort=2181
EOF
/mnt/zookeeper/bin/zkServer.sh start
/usr/bin/kafka_2.12-2.8.1/bin/kafka-server-start.sh /usr/bin/kafka_2.12-2.8.1/config/server.properties &
```
### mysql8.0.27
```
#!/usr/bin/env bash
#################################========================================================================
VERSION=8.0.27
#安装相关依赖包
yum --help > /dev/null 2>&1
if [[ $? == 0 ]];then
yum install ncurses numactl ncurses-devel libaio-devel openssl openssl-devel -y
else
sudo apt-get -y install libaio-dev
fi
#<==创建mysql用户。
useradd mysql -s /sbin/nologin -M
id mysql
if [ -f mysql-$VERSION-linux-glibc2.12-x86_64.tar.xz ];then
mv mysql-$VERSION-linux-glibc2.12-x86_64.tar.xz /opt/
tar xf /opt/mysql-$VERSION-linux-glibc2.12-x86_64.tar.xz -C /opt/
else
cd /opt && wget https://mirrors.aliyun.com/mysql/MySQL-8.0/mysql-$VERSION-linux-glibc2.12-x86_64.tar.xz && tar xf mysql-$VERSION-linux-glibc2.12-x86_64.tar.xz -C /opt/
fi
ln -s /opt/mysql-$VERSION-linux-glibc2.12-x86_64 /usr/local/mysql
ls -l /usr/local/mysql
#1)增加简易配置文件，后面在优化
cat>/etc/my.cnf<<'EOF'
[mysqld]
user=mysql
basedir=/usr/local/mysql
datadir=/data/3306/data
port=3306
socket=/tmp/mysql.sock
[client]
socket=/tmp/mysql.sock
EOF
chown mysql.mysql /etc/my.cnf
#2)创建MySQL数据目录并授权
mkdir -p /data/3306/data
chown -R mysql.mysql /data
ls -ld /data
#1) 配置PATH环境变量
echo 'export PATH=/usr/local/mysql/bin:$PATH' >>/etc/profile
. /etc/profile
#2)初始化MySQL数据库
/usr/local/mysql/bin/mysqld --initialize-insecure --user=mysql --basedir=/usr/local/mysql --datadir=/data/3306/data
#--initialize和--initialize-insecure区别
#--initialize: 会生成一个12位,4种密码复杂度临时管理员密码. 第一次登陆需要修改.
#--initialize-insecure：管理员密码为空.
#设置MySQL启动脚本
cd /usr/local/mysql/support-files/ && cp mysql.server /etc/init.d/mysqld
systemctl enable mysqld
systemctl start mysqld
```
### mysql5.7
```bash
#!/bin/bash
yum -y install numactl-libs libaio wget > /dev/null 2>&1 || apt -y install libaio1 libnuma1 wget > /dev/null 2>&1
if [[ ! -f mysql-5.7.36-el7-x86_64.tar.gz ]];then
wget https://mirrors.aliyun.com/mysql/MySQL-5.7/mysql-5.7.36-el7-x86_64.tar.gz
fi
tar -xf mysql-5.7.36-el7-x86_64.tar.gz -C /usr/local/
if [[ $? != 0 ]];then
echo "not found mysql-5.7.36-el7-x86_64.tar.gz" && exit 1
fi
mv /usr/local/mysql-5.7.36-el7-x86_64 /usr/local/mysql
#创建mysql 用户组和用户
groupadd mysql
useradd -r -g mysql mysql
#更改mysql目录下所有文件夹所属的用户组、用户以及权限
chown -R mysql:mysql /usr/local/mysql
chmod -R 755 /usr/local/mysql
#进入/usr/local/mysql/bin/目录，编译安装并初始化mysql
cd /usr/local/mysql/bin/
#重点知识2: --initialize和--initialize-insecure区别
#--initialize: 会生成一个12位,4种密码复杂度临时管理员密码. 第一次登陆需要修改.
#--initialize-insecure：管理员密码为空.
/usr/local/mysql/bin/mysqld --initialize-insecure --user=mysql --datadir=/usr/local/mysql/data --basedir=/usr/local/mysql
#编写etc目录下的my.cnf 配置文件，并添加配置
cat > /etc/my.cnf <<EOF
[mysqld]
datadir=/usr/local/mysql/data
port = 3306
sql_mode=NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES
symbolic-links=0
max_connections=400
innodb_file_per_table=1
#表名大小写不明感，敏感为
lower_case_table_names=1
EOF
chmod -R 775 /etc/my.cnf
#添加软连接，并重启mysql 服务
ln -s /usr/local/mysql/support-files/mysql.server /etc/init.d/mysql
ln -s /usr/local/mysql/bin/mysql /usr/bin/mysql
#启动mysql服务
cat > /etc/systemd/system/mysql.service <<EOF
[Unit]
Description=MySQL Server
After=network.target
[Service]
ExecStart=/etc/init.d/mysql start
ExecStop=/etc/init.d/mysql stop
ExecReload=/etc/init.d/mysql reload
Type=forking
Restart=always
TimeoutSec=30
[Install]
WantedBy=multi-user.target
EOF
systemctl enable mysql --now
#开放远程连接
mysql -e "use mysql; update user set user.Host='%' where user.User='root'; flush privileges;"
```
### 安装docker
```
#!/bin/bash
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install -y docker-ce
mkdir -p /etc/docker/
systemctl start docker
systemctl enable docker
cat >> /etc/docker/daemon.json <<EOF
{
"registry-mirrors" : [ "https://8xpk5wnt.mirror.aliyuncs.com" ]
}
EOF
sleep 1
systemctl restart docker
if [ -f "/usr/bin/docker" ];
then
echo "docker $(docker version|grep Version)安装成功"
else
echo "安装失败"
fi
```
---
### 安装go
```
#!/bin/bash
VERSION=1.17.4
go_install(){
wget https://golang.google.cn/dl/go${VERSION}.linux-amd64.tar.gz
tar -zxf go${VERSION}.linux-amd64.tar.gz -C /usr/local #解压go包到/usr/local
cat >> /etc/profile <<EOF
export GOPROXY=https://goproxy.cn,direct
export GO111MODULE=on
export PATH=\$PATH:/usr/local/go/bin:/root/gopath/bin
EOF
source /etc/profile
ln -s /usr/local/go/bin/go /usr/bin/
}
go_install
go version
if [ $? -eq 0 ];then
go env -w GO111MODULE=on
go env -w GOPROXY=https://goproxy.cn,direct
echo "golang${VERSION}安装完成"
else
echo "golang安装失败！"
fi
```
### python3.7
```
#!/bin/bash
yum install git gcc gcc-c++ make automake autoconf libtool pcre pcre-devel zlib zlib-devel openssl-devel wget vim -y
yum install libXcomposite libXcursor libXi libXtst libXrandr alsa-lib mesa-libEGL libXdamage mesa-libGL libXScrnSaver -y
yum install libffi-devel -y
wget https://www.python.org/ftp/python/3.7.2/Python-3.7.2.tar.xz
xz -d Python-3.7.2.tar.xz
tar -xvf Python-3.7.2.tar
./Python-3.7.2/configure --enable-shared --prefix=/usr/local CFLAGS=-fPIC LDFLAGS="-Wl,-rpath /usr/local/lib"
make -j24
make install
ln -sf /usr/local/bin/python3.7 /usr/bin/python3
ln -sf /usr/local/bin/python3.7 /usr/bin/python3.7
ln -sf /usr/local/bin/pip3.7 /usr/bin/pip3
ln -sf /usr/local/bin/pip3.7 /usr/bin/pip3.7
python3 --version
```
### java安装 tmp.link
```
mkdir -p /usr/local/java
mv jdk-8u301-linux-x64.tar.gz /usr/local/java/
cd /usr/local/java
tar -zxvf jdk-8u301-linux-x64.tar.gz
echo 'export JAVA_HOME=/usr/local/java/jdk1.8.0_301
export JRE_HOME=/usr/local/java/jdk1.8.0_301/jre
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib:$CLASSPATH
export PATH=$JAVA_HOME/bin:$PATH' >> /etc/profile
sleep 1
source /etc/profile
```
### 分库分表备份
```
#!/bin/bash
MYUSER=root
MYPWD=chaoge999
DBPATH=/mysql_db_back/
MYCMD="mysql -u$MYUSER -p$MYPWD"
MYDUMP="mysqldump -u$MYUSER -p$MYPWD"
[ ! -d "$DBPATH" ] && mkdir $DBPATH
for dbname in `$MYCMD -e "show databases;"|sed '1d'|egrep -v "mysql|schema"`
do
# 创建数据库同名文件夹
mkdir ${DBPATH}/${dbname}_$(date +%F) -p
# 循环找出所有数据表
for table in `$MYCMD -e "show tables from $dbname;"|sed '1d'`
do
$MYDUMP $dbname $table|gzip > $DBPATH/${dbname}_$(date +%F)/${dbname}_${table}.sql.gz
done
done
```
恢复
```
gunzip < ${dbname}_${table}.sql.gz | mysql -u$MYUSER -p$MYPWD database_name
```
```bash
#!/bin/bash
DB_HOST="10.1.125.39"
DB_USER="root"
DB_PASSWD="ssqj@easyviews.pw"
DB_PORT="30306"
DBPATH=/mysql_db_back/
DB_CMD="mysql -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} -p${DB_PASSWD}"
DB_DUMP="mysqldump -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} -p${DB_PASSWD}"
[ ! -d "$DBPATH" ] && mkdir $DBPATH
########################################################################
# Usage: mysqldump [OPTIONS] database [tables]
# OR mysqldump [OPTIONS] --databases [OPTIONS] DB1 [DB2 DB3...]
# OR mysqldump [OPTIONS] --all-databases [OPTIONS]
# For more options, use mysqldump --help
########################################################################
NC='\033[0m'
GREEN='\033[0;32m'
RED='\033[0;31m'
for DB in `${DB_CMD} -e "show databases;" |sed '1d'|egrep -v "mysql|schema" `;
do
# 创建数据库同名文件夹
mkdir ${DBPATH}/${DB}_$(date +%F) -p
# 备份数据库
$DB_DUMP --databases ${DB} |gzip > ${DBPATH}/${DB}_$(date +%F)/${DB}_`date +%F`.sql.gz
if [[ $? == 0 ]];then
echo -e "${GREEN} [`date +%Y-%m-%d_%H:%M:%S`] Database ${DB} backup completed !${NC}"
else
echo -e "${RED} [`date +%Y-%m-%d_%H:%M:%S`] Database ${DB} backup failed !${NC}" && exit 1
fi
# 遍历数据库中的表
for table in `$DB_CMD -e "show tables from ${DB};"|sed '1d'`
do
SAVE_PATH=$DBPATH/${DB}_$(date +%F)/${DB}_${table}.sql.gz
# 备份数据库中的表
$DB_DUMP ${DB} ${table}|gzip > $SAVE_PATH
if [ $? -eq 0 ]; then
echo -e "${GREEN} Database: [${DB}] - Table:[${table}] backup to $SAVE_PATH completed ! ${NC}"
else
echo -e "${RED} Database: [${DB}] - Table:[${table}] backup failed ! ${NC}" && exit 1
fi
done
done
```