首页 / GIT / gitlab多线程备份脚本
gitlab多线程备份脚本
内容导读
互联网集市收集整理的这篇技术教程文章主要介绍了gitlab多线程备份脚本，小编现在分享给大家，供广大互联网技能从业者学习和参考。文章包含7380字，纯文字阅读大概需要11分钟。
内容图文
将公司的gitlab服务器进行备份，并且将备份rsync到异地机房，因为备份文件太大，所以这里做了切割成多个文件，进行多线程传输。
#!/usr/bin/env perl
################
#filename: gitback.pl
#version : 1.0
#author : zzq
#date : 20160705
#function: The transmission of gitlab backup files to the remote host
#########################
use strict;
use threads;
use Thread::Queue;
use File::Rsync;
use threads::shared;
use File::Find;
use File::Basename;
use POSIX qw(strftime);
use Log::Log4perl qw(:easy);
use Mail::Sender;
my ($logger,$Msg,$Subject,$filename,$srcFile,$fileName,$filePath,$fileSize,$backFilename);
my $fileCount;
my $filemsg;
my $message;
######### remote host rsync config ##################
my $remotehost = ‘rsync@10.xx.xx.xx::gitbackup‘;
############# back file dir##############
my $backFiledir = ‘/data/gitbackups/‘;
######## max threads count##############
my $thread_max = 10;
########## current time #############
my $currentTime = strftime("%Y%m%d%H%M", localtime(time));
$Msg = "$currentTime\n";
########## log file ##############
my $logFile = "gitback.log";
my $splitFileQueue = Thread::Queue->new();
my $remotefileQueue = Thread::Queue->new();
############## define smtp ######################
#Sender
my $send_user=‘monitor@.cc.cn‘;
#passwd for Sender
#my $send_user_pw=‘123‘;
#smtp address
my $smtp_server=‘10.168.xx.xx‘;
#define mail subject
$Subject = "gitback-$currentTime";
#address
my @address=qw(xxx.cn);
#################################################
################### define log conf #########################
my $log_logger = get_logger();
$log_logger->level($INFO);
my $file_appender = Log::Log4perl::Appender->new(
"Log::Dispatch::File",
filename => "$logFile",
mode => "append",
);
$log_logger->add_appender($file_appender);
my $layout = Log::Log4perl::Layout::PatternLayout->new(
"%d %p > %F{1} %M -- %m%n"
);
$file_appender->layout($layout);
###################################################
main();
sub main {
Usage() unless ( scalar(@ARGV) == 0);
$log_logger->info("Begin:$currentTime");
$srcFile = backup();
print "srcFile is $srcFile\n";
($fileName,$filePath) = fileparse($srcFile) if (-f $srcFile);
$fileSize=(stat $srcFile)[7];
srcFileMd5($srcFile);
splitFile($srcFile);
find(\&findSplitFile,$filePath);
$fileCount = $splitFileQueue->pending();
$Msg = "$Msg"."FileCount: $fileCount\n"."$filemsg";
splitFileMd5Thread();
rsyncThread();
for my $addr (@address){
sendMail($addr);
}
$currentTime=strftime("%Y%m%d%H%M", localtime(time));
$log_logger->info("End:$currentTime");
}
###################Access to the backup name for gitlab ################
sub backup {
my $result=`gitlab-rake gitlab:backup:create`;
my $flag=`echo $?`;
if ($flag == 0){
$log_logger->info("gitlab-backup-create : success ");
my @result=split(/\n/,$result);
for my $cc (@result){
if ($cc =~ /(?<filename>\d+_gitlab_backup.tar)/){
$backFilename=$+{‘filename‘};
$log_logger->info("gitlab-backup-filename : $backFilename");
}
}
}else{
$log_logger->info("gitlab-backup-create : fail ");
}
return "$backFiledir$backFilename";
}
############# Search the file after segmentation ################
sub findSplitFile {
if ( $_ =~ /$fileName\.\d{3}$/) {
$log_logger->info("splitFileQueue enqueue : $_");
$filemsg = "$filemsg"."$_\n";
$splitFileQueue->enqueue($_);
$log_logger->info("remotefileQueue enqueue : $_");
$remotefileQueue->enqueue($_);
# $log_logger->info("print to logfile : $_");
}
}
############## Search the MD5 file after segmentation ############
sub splitFileMd5Thread {
while($splitFileQueue->pending()) {
my $Queue = $splitFileQueue->dequeue();
$log_logger->info("splitfileQueueMD5 dequeue : $Queue ");
threads->create(\&splitFileMd5,$Queue);
}
foreach my $thread (threads->list(threads::all)){
if ($thread->is_joinable()){
$thread->join();
}
}
foreach my $thread ( threads->list(threads::all) ) {
$thread->join();
}
}
################## Search for the source file md5 #################
sub srcFileMd5 {
my $file=shift;
my $md5_command=`md5sum $file`;
my $flag=`echo $?`;
if ($flag == 0) {
my ($file_md5,$filename) = split(/\s+/,$md5_command);
my $log = "srcfile:$file filesize:$fileSize file_md5sum:$file_md5";
$Msg = "$Msg"."srcfile:$file filesize:$fileSize file_md5sum:$file_md5\n";
$log_logger->info($log);
}else {
my $file_md5 = "system command maybe exec fail, command result code is $md5_command";
my $log = "srcfile:$file filesize:$fileSize file_md5sum:$file_md5";
$Msg = "$Msg"."srcfile:$file filesize:$fileSize file_md5sum:$file_md5\n";
$log_logger->warn($log);
}
}
####### split File MD5 ###############
sub splitFileMd5 {
my $file=shift;
my $md5_command=`md5sum $filePath$file`;
my $flag=`echo $?`;
if ($flag == 0) {
my ($file_md5,$filename) = split(/\s+/,$md5_command);
my $log = "file:$file file_md5sum:$file_md5";
$log_logger->info($log);
}
}
################# Will be cut by the source file ################
sub splitFile {
my $srcFile = shift;
my $split_command =`split -b 50M $srcFile -d -a 3 $srcFile.`;
$split_command = `echo $? `;
if ($split_command == 0 ){
my $splitFileCode = ‘success‘;
$log_logger->info(" srcFile:$srcFile desc:$splitFileCode splitFileResult:$split_command");
}else{
my $splitFileCode = ‘fail‘;
$log_logger->warn(" srcFile:$srcFile desc:$splitFileCode splitFileResult:$split_command");
}
}
sub rsyncThread {
while ( $remotefileQueue->pending() ){
if (scalar threads->list() < $thread_max) {
my $readQueue = $remotefileQueue->dequeue();
$log_logger->info("remotefileQueue dequeue : $readQueue ");
threads->create(\&rsync,$readQueue);
}
foreach my $thread (threads->list(threads::all)){
if ($thread->is_joinable()){
$thread->join();
}
}
}
foreach my $thread ( threads->list(threads::all) ) {
$thread->join();
}
}
sub rsync {
my $file = shift;
my $obj = File::Rsync->new(
{
archive => 1,
compress => 1,
checksum => 1,
recursive => 1,
owner => 1,
devices => 1,
group => 1,
perms => 1,
times => 1,
verbose => 1,
progress => 1,
stats => 1,
links => 1,
‘hard-links‘ => 1,
‘ignore-times‘ => 1,
‘password-file‘ => ‘./rsync.pass‘,
}
);
$obj->exec( { src => "$filePath$file", dest => $remotehost } );
#or warn ($log_logger->warn("rsync: $file to $remotehost fail!" ));
my $rval = $obj->realstatus;
if ($rval == 0 ) {
$log_logger->info("rsync: $file to $remotehost Success!\n");
}else{
my $rsyncError = sprintf ("%s", $obj->err);
$message = "$message"."Transfer Failed:\nfile:$file reason:$rsyncError";
$log_logger->info("rsync: $file to $remotehost fail reason:$rsyncError");
}
}
sub sendMail {
my $CONTACTEMAIL = shift;
my $sender = new Mail::Sender{
ctype => ‘text/plain; charset=utf-8‘,
encoding => ‘utf-8‘,
} ;
die "Error in mailing : $Mail::Sender::Error\n" unless ref $sender;
if ($sender->MailMsg({
smtp => $smtp_server,
from => $send_user,
to => $CONTACTEMAIL,
subject => $Subject,
msg => $Msg,
# file => "$file",
# auth => ‘LOGIN‘,
# authid => $send_user,
# authpwd => $send_user_pw,
charset=>‘utf-8‘
}) < 0) {
die $log_logger->error("senermail fail -- $Mail::Sender::Error\n");
}
$sender->Close();
}
sub Usage {
print <<"END";
perl gitback.pl
END
exit 2;
}
本文出自 “大風” 博客，请务必保留此出处http://lansgg.blog.51cto.com/5675165/1864156
原文：http://lansgg.blog.51cto.com/5675165/1864156
内容总结
以上是互联网集市为您收集整理的gitlab多线程备份脚本全部内容，希望文章能够帮你解决gitlab多线程备份脚本所遇到的程序开发问题。 如果觉得互联网集市技术教程内容还不错，欢迎将互联网集市网站推荐给程序员好友。
内容备注
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 gblab@vip.qq.com 举报，一经查实，本站将立刻删除。
内容手机端
扫描二维码推送至手机访问。
来源：【匿名】