nginx+tomcat高可用、高性能jsp集群

#!/bin/bash

# BY kerryhu

# MAIL:king_819@https://www.360docs.net/doc/1f15185794.html,

# BLOG:https://www.360docs.net/doc/1f15185794.html,

# Please manual operation yum of before Operation.....

#yum -y install gcc gcc-c++ autoconf libjpeg libjpeg-devel libpng libpng-devel \

#freetype freetype-devel libxml2 libxml2-devel zlib zlib-devel glibc glibc-devel \

#glib2 glib2-devel bzip2 bzip2-devel ncurses ncurses-devel curl

curl-devel e2fsprogs \

#e2fsprogs-devel krb5 krb5-devel libidn libidn-devel openssl

openssl-devel openldap \

#openldap-devel nss_ldap openldap-clients openldap-servers

一、网络拓扑:

lvs-master: 192.168.9.201

LVS-backup: 192.168.9.202

LVS-VIP: 192.168.9.200

web_1: 192.168.9.203

web_2: 192.168.9.204

目的：搭建高可用、高性能的jsp集群

二、初始化系统

#init system

系统环境: CentOS 5.5（定制安装）

组件：

Base

Development Libraries

Development Tools

Editors

Text-based Internet

./init_system.sh #此脚本参见

https://www.360docs.net/doc/1f15185794.html,/172631/555535

三、LVS+keeplived

#关于LVS+keeplived的配置请参考我的另一篇博文《CentOS5.5环境下布署LVS+keepalived 》https://www.360docs.net/doc/1f15185794.html,/172631/401253

四、nginx、tomcat整合

#config web_1、web_2

#download software

cd /opt

wget

https://www.360docs.net/doc/1f15185794.html,/apache-mirror/tomcat/tomcat-6/v6.0.32/bin/apac he-tomcat-6.0.32.tar.gz

wget

https://www.360docs.net/doc/1f15185794.html,/otn-pub/java/jdk/6u25-b06/jdk-6u25-linux-i 586.bin

wget https://www.360docs.net/doc/1f15185794.html,/download/nginx-1.0.0.tar.gz

wget https://www.360docs.net/doc/1f15185794.html,/pcre/pcre-8.12.tar.gz

wget

https://www.360docs.net/doc/1f15185794.html,//tomcat/tomcat-connectors/native/1.1.20/sourc e/tomcat-native-1.1.20-src.tar.gz

wget https://www.360docs.net/doc/1f15185794.html,/dist/apr/apr-1.4.2.tar.gz

wget https://www.360docs.net/doc/1f15185794.html,/dist/apr/apr-util-1.3.9.tar.gz

#================================ 1、安装tomcat、JDK

=================================

mv apache-tomcat-6.0.32.tar.gz jdk-6u25-linux-i586.bin /usr/local/ tar -zxvf apache-tomcat-6.0.32.tar.gz

mv apache-tomcat-6.0.32 tomcat

chmod +x jdk-6u25-linux-i586.bin

./jdk-6u25-linux-i586.bin

#apr 与 tomcat-native 提供更好的伸缩性、性能和集成到本地服务器技术,如果没有apr技术，启动tomcat 时出现如下提示

#INFO: The APR based Apache Tomcat Native library which allows optimal performance in production environments was not found on the

java.library.path:

/usr/local/jdk1.6.0_25/jre/lib/i386/client:/usr/local/jdk1.6.0_25/jre /lib/i386:/usr/local/jdk1.6.0_25/jre/../lib/i386:/usr/java/packages/l ib/i386:/lib:/usr/lib

rpm -e --nodeps apr-1.2.7-11.el5_3.1

rpm -e --nodeps apr-1.2.7-11.el5_3.1

tar -zxvf apr-1.4.2.tar.gz

cd apr-1.4.2

./configure

make;make install

cd ../

tar -zxvf apr-util-1.3.9.tar.gz

cd apr-util-1.3.9

./configure --with-apr=/usr/local/apr

make;make install

cd ../

cp tomcat-native-1.1.20-src.tar.gz /usr/local/tomcat/bin/

cd /usr/local/tomcat/bin

tar -zxvf tomcat-native.tar.gz

cd ./tomcat-native-1.1.12-src/jni/native/

./configure --with-apr=/usr/local/apr

--with-java-home=/usr/local/jdk1.6.0_25

make;make install

#设置环境变量

cat >>/etc/profile<

export JAVA_HOME=/usr/local/jdk1.6.0_25

export CLASS_PATH=$JAVA_HOME/lib:$JAVA_HOME/jre/lib

export PATH=.:$PATH:$JAVA_HOME/bin

export CATALINA_HOME=/usr/local/tomcat

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/apr/lib

EOF

source /etc/profile

#查看java版本

java -version

#启动tomcat

/usr/local/tomcat/bin/startup.sh

echo "/usr/local/tomcat/bin/startup.sh" >> /etc/rc.local

#将tomcat添加到系统服务

cd /opt

wget

http://forja.cenatic.es/plugins/scmsvn/viewcvs.php/*checkout*/servers /tomcat/bin/jsvc.tar.gz?root=norma&rev=82&pathrev=82

mv jsvc.tar.gz /usr/local/tomcat/bin/

tar -zxvf jsvc.tar.gz

cd jsvc-src

chmod +x configure

./configure --with-java=/usr/local/jdk1.6.0_25

make

cp /usr/local/tomcat/bin/jsvc-src/native/Tomcat5.sh

/etc/rc.d/init.d/tomcat

chmod +x /etc/rc.d/init.d/tomcat

vi /etc/rc.d/init.d/tomcat

#chkconfig: 2345 85 15

#description: apache-tomcat.6 #这两行在#!/bin/sh下面添加

JAVA_HOME=/usr/local/jdk1.6.0_25

CATALINA_HOME=/usr/local/tomcat

DAEMON_HOME=/usr/local/tomcat/bin

TOMCAT_USER=tomcat

CATALINA_BASE=/usr/local/tomcat

CATALINA_OPTS="-Djava.library.path=/usr/local/apr/lib"

$DAEMON_HOME/jsvc-src/jsvc #有两处需要修改

#添加tomcat用户

groupadd tomcat

useradd -g tomcat -s /sbin/nologin -M tomcat

chown -R tomcat.tomcat /usr/local/tomcat

chkconfig --add tomcat

chkconfig --level 2345 tomcat on

#http://localhost:8080，如果看到猫的页面即tomcat和jdk安装成功

#新建文件目录/data/www为网站存放目录，设置server.xml文件，在Host name="localhost"处将appBase=的指向路径改为/data/www/test,创建index.jsp至/data/www /test/ROOT，内容为：“My web!”

#修改tomcat网站目录

#sed -i 's#appBase="webapps"#appBase="/data/www/test"#'

/usr/local/tomcat/conf/server.xml

#报错信息：

https://www.360docs.net/doc/1f15185794.html,ng.IllegalArgumentException: Document base /data/www/tru boshop/host-manager does not exist or is not a readable directory #解决方法：删除conf\Catalina\localhost目录下面的host-manager.xml、manager.xml两个文件

#============================= 2、安装Nginx

=============================

/usr/sbin/groupadd www

/usr/sbin/useradd -g www www -s /sbin/nologin

mkdir -p /data/www

chmod +w /data/www

chown -R www:www /data/www

tar zxvf pcre-8.12.tar.gz

cd pcre-8.12/

./configure

make;make install

cd ../

#使用Tcmalloc 优化nginx、mysql

wget

https://www.360docs.net/doc/1f15185794.html,/files/google-perftools-1.7.tar .gz

tar -zxvf google-perftools-1.7.tar.gz

cd google-perftools-1.7/

./configure

make;make install

echo "/usr/local/lib" > /etc/ld.so.conf.d/usr_local_lib.conf

/sbin/ldconfig

cd ../

tar zxvf nginx-1.0.0.tar.gz

cd nginx-1.0.0/

./configure --user=www --group=www --prefix=/usr/local/nginx

--with-http_stub_status_module --with-http_ssl_module

--with-http_flv_module --with-http_gzip_static_module

--with-google_perftools_module

make;make install

cd ../

mkdir -p /data/logs

chmod +w /data/logs

chown -R www:www /data/logs

#编辑配置文件

#编辑nginx.config

mv /usr/local/nginx/conf/nginx.conf

/usr/local/nginx/conf/nginx.conf.bak

cat >>/usr/local/nginx/conf/nginx.conf<

user www www;

worker_processes 8;

error_log /usr/local/nginx/logs/nginx_error.log crit;

pid /usr/local/nginx/nginx.pid;

#使用Tcmalloc优化nginx性能

google_perftools_profiles /var/tmp/tcmalloc;

#Specifies the value for maximum file descriptors that can be opened by this process.

worker_rlimit_nofile 65535;

#工作模式及连接数上限

events

{

use epoll;

worker_connections 65535;

}

#设定http服务器，利用它的反向代理功能提供负载均衡支持

http

{

#设定mime类型

include mime.types;

default_type application/octet-stream;

#charset gb2312;

#设定请求缓冲

server_names_hash_bucket_size 128;

client_header_buffer_size 32k;

large_client_header_buffers 4 32k;

client_max_body_size 300m;

sendfile on;

tcp_nopush on;

keepalive_timeout 60;

tcp_nodelay on;

server_tokens off;

client_body_buffer_size 512k;

proxy_connect_timeout 5;

proxy_send_timeout 60;

proxy_read_timeout 5;

proxy_buffer_size 16k;

proxy_buffers 4 64k;

proxy_busy_buffers_size 128k;

proxy_temp_file_write_size 128k;

# fastcgi_connect_timeout 300;

# fastcgi_send_timeout 300;

# fastcgi_read_timeout 300;

# fastcgi_buffer_size 64k;

# fastcgi_buffers 4 64k;

# fastcgi_busy_buffers_size 128k;

# fastcgi_temp_file_write_size 128k;

gzip on;

gzip_min_length 1k;

gzip_buffers 4 16k;

gzip_http_version 1.1;

gzip_comp_level 2;

gzip_types text/plain application/x-javascript text/css application/xml;

gzip_vary on;

#limit_zone crawler $binary_remote_addr 10m;

###禁止通过ip访问站点

server{

server_name _;

return 404;

}

server

{

listen 80;

server_name https://www.360docs.net/doc/1f15185794.html,;

index index.html index.htm index.jsp index.do;#设定访问的默认首页地址

root /data/www/shopxx;#设定网站的资源存放路径

#limit_conn crawler 20;

if (-d $request_filename)

{

rewrite ^/(.*)([^/])$ http://$host/$1$2/ permanent;

}

#所有jsp的页面均交由tomcat处理

location ~ \.(jsp|jspx|do)?$ {

proxy_set_header Host $host;

proxy_set_header X-Real-IP $remote_addr;

proxy_pass http://127.0.0.1:8080;#转向tomcat处理

}

location

~ .*\.(htm|html|gif|jpg|jpeg|png|bmp|swf|ioc|rar|zip|txt|flv|mid|doc| ppt|pdf|xls|mp3|wma)$ #设定访问静态文件直接读取不经过tomcat

{

expires 30d;

}

location ~ .*\.(js|css)?$

{

expires 1h;

}

#定义访问日志的写入格式

log_format wwwlog '$remote_addr - $remote_user [$time_local] "$request" '

'$status $body_bytes_sent "$http_referer" '

'"$http_user_agent" $http_x_forwarded_for';

access_log /data/logs/www_nginx.log wwwlog;#设定访问日志的存放路径 }

server

{

listen 80;

server_name https://www.360docs.net/doc/1f15185794.html,;

index index.html index.htm index.jsp index.do;#设定访问的默认首页地址

root /data/www/jsprun;#设定网站的资源存放路径

#limit_conn crawler 20;

if (-d $request_filename)

{

rewrite ^/(.*)([^/])$ http://$host/$1$2/ permanent;

}

#所有jsp的页面均交由tomcat处理

location ~ \.(jsp|jspx|do)?$ {

proxy_set_header Host $host;

proxy_set_header X-Real-IP $remote_addr;

proxy_pass http://127.0.0.1:8080;#转向tomcat处理

}

location

~ .*\.(htm|html|gif|jpg|jpeg|png|bmp|swf|ioc|rar|zip|txt|flv|mid|doc| ppt|pdf|xls|mp3|wma)$ #设定访问静态文件直接读取不经过tomcat

{

expires 30d;

}

location ~ .*\.(js|css)?$

{

expires 1h;

}

#定义访问日志的写入格式

log_format bbslog '$remote_addr - $remote_user [$time_local] "$request" '

'$status $body_bytes_sent "$http_referer" '

'"$http_user_agent" $http_x_forwarded_for';

access_log /data/logs/bbs_nginx.log bbslog;#设定访问日志的存放路径 }

server

{

listen 80;

server_name https://www.360docs.net/doc/1f15185794.html,;

location / {

stub_status on;

access_log off;

}

}

}

EOF

#检测配置文件

#/usr/local/nginx/sbin/nginx -t

#启动nginx

/usr/local/nginx/sbin/nginx

echo "/usr/local/nginx/sbin/nginx" >> /etc/rc.local

#查看nginx主进程号

#ps -ef | grep "nginx: master process" | grep -v "grep" | awk -F ' ' '{print $2}'

#停止nginx

#/usr/local/nginx/sbin/nginx -s stop

#平滑重启

#/usr/local/nginx/sbin/nginx -s reload

#============================= 3、配置一套java商城系统，进行测试

=============================

cd /opt

wget https://www.360docs.net/doc/1f15185794.html,/rar/rarlinux-3.8.0.tar.gz

#rar解压

tar -zxvf rarlinux-3.8.0.tar.gz

cd rar

make install

cd /opt

unrar x jsprun.rar

mv jsprun /data/www/

#7z解压

wget

https://www.360docs.net/doc/1f15185794.html,/projects/p7zip/files/p7zip/9.13/p7zip_9.13_src _all.tar.bz2/download

tar -jxvf p7zip_9.13_src_all.tar.bz2

cd p7zip_9.13

make;make install

7za x shopxx.7z

mv shopxx /data/www/

#配置tomcat虚拟主机

vi /usr/local/tomcat/conf/server.xml

unpackWARs="true" autoDeploy="true">

unpackWARs="true" autoDeploy="true">

#使用cmake编译安装mysql

cd /opt

wget https://www.360docs.net/doc/1f15185794.html,/files/v2.8/cmake-2.8.4.tar.gz

wget https://www.360docs.net/doc/1f15185794.html,/mysql/MySQL-5.5/mysql-5.5.10.tar.gz

tar -zxvf cmake-2.8.4.tar.gz

cd cmake-2.8.4

./configu

make;make install

/usr/sbin/groupadd mysql

/usr/sbin/useradd -g mysql mysql -s /sbin/nologin

mkdir -p /data/mysql/data

mkdir -p /data/mysql/binlog

mkdir -p /data/mysql/relaylog

mkdir -p /data/mysql/mysql

chown -R mysql:mysql /data/mysql

cd /opt

tar zxvf mysql-5.5.10.tar.gz

cd mysql-5.5.10/

rm -rf CMakeCache.txt

cmake .

-DCMAKE_INSTALL_PREFIX=/usr/local/mysql/ -DMYSQL_DATADIR=/data/mysql /data

-DWITH_INNOBASE_STORAGE_ENGINE=1 -DMYSQL_TCP_PORT=3306 -DEXTRA_CHAR SETS=all -DDEFAULT_CHARSET=utf8 -DDEFAULT_COLLATION=utf8_general_ci -DMYSQL_UNIX_ADDR=/data/mysql/mysql.sock -DMYSQL_USER=mysql -DWITH_ DEBUG=0

make;make install

chmod +w /usr/local/mysql

chown -R mysql:mysql /usr/local/mysql

cat >>/data/mysql/https://www.360docs.net/doc/1f15185794.html,f<

[client]

character-set-server = utf8

port = 3306

socket = /data/mysql/mysql.sock

[mysqld]

character-set-server = utf8

replicate-ignore-db = mysql

replicate-ignore-db = test

replicate-ignore-db = information_schema

user = mysql

port = 3306

socket = /data/mysql/mysql.sock

basedir = /usr/local/mysql

datadir = /data/mysql/data

log-error = /data/mysql/mysql_error.log

pid-file = /data/mysql/mysql.pid

open_files_limit = 10240

back_log = 600

max_connections = 5000

max_connect_errors = 6000

table_cache = 614

external-locking = FALSE

max_allowed_packet = 32M

sort_buffer_size = 1M

join_buffer_size = 1M

thread_cache_size = 300

thread_concurrency = 8

query_cache_size = 512M

query_cache_limit = 2M

query_cache_min_res_unit = 2k

default-storage-engine = MyISAM

thread_stack = 192K

transaction_isolation = READ-COMMITTED

tmp_table_size = 246M

max_heap_table_size = 246M

long_query_time = 3

log-slave-updates

log-bin = /data/mysql/binlog/binlog

binlog_cache_size = 4M

binlog_format = MIXED

max_binlog_cache_size = 8M

max_binlog_size = 100M

relay-log-index = /data/mysql/relaylog/relaylog

relay-log-info-file = /data/mysql/relaylog/relaylog relay-log = /data/mysql/relaylog/relaylog

expire_logs_days = 30

key_buffer_size = 256M

read_buffer_size = 1M

read_rnd_buffer_size = 16M

bulk_insert_buffer_size = 64M

myisam_sort_buffer_size = 128M

myisam_max_sort_file_size = 10G

myisam_repair_threads = 1

myisam_recover

interactive_timeout = 120

wait_timeout = 120

skip-name-resolve

master-connect-retry = 10

slave-skip-errors = 1032,1062,126,1114,1146,1048,1396

#master-host = 192.168.1.2

#master-user = username

#master-password = password

#master-port = 3306

server-id = 1

innodb_additional_mem_pool_size = 16M

innodb_buffer_pool_size = 512M

innodb_data_file_path = ibdata1:256M:autoextend

innodb_file_io_threads = 4

innodb_thread_concurrency = 8

innodb_flush_log_at_trx_commit = 2

innodb_log_buffer_size = 16M

innodb_log_file_size = 128M

innodb_log_files_in_group = 3

innodb_max_dirty_pages_pct = 90

innodb_lock_wait_timeout = 120

innodb_file_per_table = 0

#log-slow-queries = /data/mysql/slow.log

#long_query_time = 10

[mysqldump]

quick

max_allowed_packet = 32M

EOF

#初始化mysql

/usr/local/mysql/scripts/mysql_install_db

-–defaults-file=/data/mysql/https://www.360docs.net/doc/1f15185794.html,f --basedir=/usr/local/mysql --datadir=/data/mysql/data --user=mysql

#利用TCMalloc提高mysql在高并发下的性能

vi /usr/local/mysql/bin/mysqld_safe

#在# executing mysqld_safe的下一行，加上：

export LD_PRELOAD=/usr/local/lib/libtcmalloc.so

#使用lsof命令查看tcmalloc是否起效

/usr/sbin/lsof -n | grep tcmalloc

#设置mysql启动文件

cp support-files/mysql.server /etc/rc.d/init.d/mysqld

vi /etc/rc.d/init.d/mysqld

basedir=/usr/local/mysql

datadir=/data/mysql/data

chmod 700 /etc/rc.d/init.d/mysqld

/etc/rc.d/init.d/mysqld start

/sbin/chkconfig --add mysqld

/sbin/chkconfig --level 2345 mysqld on

ln -s /usr/local/mysql/bin/mysql /sbin/mysql

ln -s /usr/local/mysql/bin/mysqladmin /sbin/mysqladmin

#设置root密码（753951）

/sbin/mysqladmin -u root password 753951

#配置库文件搜索路径

echo "/usr/local/mysql/lib/mysql" >> /etc/ld.so.conf

/sbin/ldconfig

#添加/usr/local/mysql/bin到环境变量PATH中

export PATH=$PATH:/usr/local/mysql/bin

#添加mysql管理帐户

#mysql -h localhost -u root -p753951

#msqyl> use mysql;

#msqyl> grant all on *.* to 'kerry'@'192.168.9.100' identified by

'852741';

#msqyl> flush privileges;

#msqyl> exit;

#创建一个shopxx空库

CREATE DATABASE shopxx DEFAULT CHARACTER SET utf8 DEFAULT COLLATE

utf8_general_ci;

#=============================== 4、tomcat优

化 ============================

#添加tomcat管理员

#Tomcat 6.x不再提供admin管理工具(Tomcat's administration web application is no longer installed by default，Download and install the "admin" package to use it)，要自己安装

cd /opt

wget

https://www.360docs.net/doc/1f15185794.html,/dist/tomcat/tomcat-5/v5.5.26/bin/apache-tom cat-5.5.26-admin.tar.gz

wget

https://www.360docs.net/doc/1f15185794.html,/apache//commons/logging/binaries/commons-lo gging-1.1.1-bin.tar.gz

wget

https://www.360docs.net/doc/1f15185794.html,/apache-mirror//commons/modeler/binaries/common s-modeler-2.0.1.tar.gz

tar -zxvf apache-tomcat-5.5.26-admin.tar.gz

cd apache-tomcat-5.5.26

cp -r conf/Catalina /usr/local/tomcat/conf/

cp -r server /usr/local/tomcat/

cd /opt

tar -zxvf commons-modeler-2.0.1.tar.gz

cd commons-modeler-2.0.1

cp -r *.jar /usr/local/tomcat/lib/

cd ../

tar -zxvf commons-logging-1.1.1-bin.tar.gz

cd commons-logging-1.1.1

cp -r *.jar /usr/local/tomcat/lib/

vi /usr/local/tomcat/conf/tomcat-users.xml

#在 中间添加 ,并去掉注释

vi /usr/local/tomcat/conf/context.xml

docBase="/usr/local/tomcat/webapps/manager" />

#修改Java虚拟机的虚拟内存，tomcat默认的JVM比较小，查看JVM，打开http://127.0.0.1:8080/，点击"status"，查看JVM选项

#在修改配置文件的时候不知道怎么把manager/WEB-INF/web.xml搞没见了，导致在访问管理页面的时候出现“The requested resource (/manager/s tatus) is not available.”

Free memory: 3.25 MB Total memory: 7.62 MB Max memory: 121.81 MB

vi /etc/init.d/tomcat

#在CATALINA_OPTS="-Djava.library.path=/usr/local/apr/lib"下方添加一

句

JAVA_OPTS="-Xms1024m -Xmx1024m -Xmn256m -Djava.awt.headless=true"

#重启一下tomcat，然后再次查看JVM

#Free memory: 953.12 MB Total memory: 998.43 MB Max memory: 998.43 MB #修改tomcat并发连接数，tomcat6.0 默认并发数为200，查看方法和JVM一样，

查看http-8080选项

Max threads: 200

vi /usr/local/tomcat/conf/server.xml

#修改前：

redirectPort="8443"

connectionTimeout="20000"

/>

#修改后：

maxHttpHeaderSize="8192"

useBodyEncodingForURI="true"

maxThreads="1000"

redirectPort="8443"

enableLookups="false"

compression="on"

compressionMinSize="2048"

compressableMimeType="text/html,text/xml,text/javascr ipt,text/css,text/plain"

connectionTimeout="20000"

disableUploadTimeout="true"

/>

port：Tomcat服务器监听的端口号

maxHttpHeaderSize：Http的Header的最大限制

enableLookups="false" 使用允许DNS查询，通常情况下设置为false redirectPort 服务器正在处理http请求时收到了一个SSL传输请求后重定向的端口号

maxThreads Tomcat使用线程来处理接收的每个请求。这个值表示Tomcat可创建的最大的线程数。

acceptCount 指定当所有可以使用的处理请求的线程数都被使用时，可以放到处理队列中的请求数，超过这个数的请求将不予处理。

connectionTimeout 网络连接超时，单位：毫秒。设置为0表示永不超时，这样设置有隐患的。通常可设置为30000毫秒。

minSpareThreads Tomcat初始化时创建的线程数。

maxSpareThreads 一旦创建的线程超过这个值，Tomcat就会关闭不再需要的socket线程。

compression="on" 打开压缩功能

compressionMinSize 启用压缩的输出内容大小，这里面默认为2KB compressableMimeType压缩类型

#重启tomcat后再次查看

Max threads: 1000