#!/usr/bin/env perl
#use warnings;
use strict;
use MIME::Base64;
use SOAP::Lite;
use Fcntl qw(:flock);
use Archive::Extract;
use XML::Simple;
use Data::Dumper;
use utf8;
use Encode;
binmode STDOUT, ":utf8";
use constant {
NOTICE => 0,
WARN => 1,
ERROR => 2,
DEBUG => 3,
};
# Отключаем буферизацию вывода
$|=1;
# Название компании по которой делаем выгрузку, файлы должны иметь вид компания.req компания.sig
my $company = shift or die("Use script with parameters\n");
my $requestFile = "/root/bin/" . $company . ".req";
my $signatureFile = "/root/bin/" . $company .".sig";
my $registryFile = "/tmp/" . $company . ".registry.zip";
my $registryTime = 0;
my $retries = 240; # Количество секунд ожидания приема файла
my $step = 30; # Опрашивать каждые step секунд
my $update = 1;
our $debug = 0;
my $aclDomainFile = "/etc/squid/block_domain.acl";
my $aclUrlFile = "/etc/squid/block_url.acl";
my $hostFile = "/etc/squid/banned_hosts";
my $request;
my $signature;
sub trim {
my $text = shift;
$text =~ s/^\s+|\s+$//g;
return $text;
}
sub getLevel {
my $level = shift;
if($level == 0){
return "NOTICE";
}
elsif($level == 1){
return "WARN";
}
elsif($level == 2){
return "ERROR";
}
return "DEBUG";
}
sub _log {
my $level = shift;
my $message = shift;
if($debug == 1 || ($level < DEBUG and $debug == 0)){
my ($second, $minute, $hour, $mday, $mon, $year, $wday, $yday, undef) = localtime(time);
$year += 1900;
$mon += 1;
printf("ddd-d:d:d : %s : %s", $year, $mon, $mday, $hour, $minute, $second, getLevel($level), $message);
}
}
# Разархивируем файл
sub unzip {
my $zip = shift;
my $archive = Archive::Extract->new( archive => $zip);
my $archive_ok = $archive->extract( to => '/tmp') or die "Cannot extract files from $zip: $archive->error";
# Пока путь возвращаем через костыль
return "/tmp/dump.xml";
}
# Пасрим XML
sub parse_xml {
my $xml = shift;
my $simple = XML::Simple->new();
my $data = $simple->XMLin($xml);
my $urls;
my $domains;
my $blockDomains;
my $ips;
my %seenDomains;
my %seenIps;
foreach my $number (keys %{$data->{'content'}}){
my $block = 0;
if(ref($data->{'content'}{$number}{'url'}) eq "ARRAY"){
foreach my $url (@{$data->{'content'}{$number}{'url'}}){
if($url =~ /^https:/i){
$block = 1;
}
push @$urls, quotemeta(trim($url));
}
}
else{
if($data->{'content'}{$number}{'url'} =~ /^https:/i){
$block = 1;
}
push @$urls, quotemeta(trim($data->{'content'}{$number}{'url'}));
}
if(! $seenDomains{$data->{'content'}{$number}{'domain'}}++){
if($block){
push @$blockDomains, $data->{'content'}{$number}{'domain'};
}
push @$domains, $data->{'content'}{$number}{'domain'};
}
if(ref($data->{'content'}{$number}{'ip'}) eq "ARRAY"){
foreach my $ip (@{$data->{'content'}{$number}{'ip'}}){
if(! $seenIps{$ip}++){
push @$ips, $ip;
}
}
}
else{
if(! $seenIps{$data->{'content'}{$number}{'ip'}}++){
push @$ips, $data->{'content'}{$number}{'ip'};
}
}
}
@$urls = sort @$urls;
@$domains = sort @$domains;
@$ips = sort @$ips;
@$blockDomains = sort @$blockDomains;
return ($urls, $domains, $ips, $blockDomains);
}
sub _ipset {
my $ips = shift;
my $ipset = "/usr/sbin/ipset";
my $ipsetName = "blockip";
my $ipsetTemp = "blocktmp";
system "$ipset create $ipsetTemp hash:ip";
foreach my $ip (@{$ips}){
system "$ipset add $ipsetTemp $ip";
}
system "$ipset swap $ipsetTemp $ipsetName";
system "$ipset destroy $ipsetTemp";
}
# Логирования, надо отключить после тестов
my $logFile = "/tmp/" . $company . ".log";
open LOGFILE, ">>$logFile" or die "Cannot open log file $logFile: $!";
# Переключаем стандартный вывод в файл логирования
select LOGFILE;
my $currentTime = time;
my ($second, $minute, $hour, undef, undef, undef, undef, undef, undef) = localtime($currentTime);
# Если файл реестра существует запоминаем время последнего редактирования
if(-e $registryFile){
_log DEBUG, "Registry file $registryFile exists\n";
$registryTime = (stat($registryFile))[9];
}
else{
_log DEBUG, "Registry file $registryFile not exists\n";
}
# 4 раза в сутки скачиваем файл реестра обязательно в независимости от последеного обновления файла и реестра
if( ($hour == 0 and ($minute >=0 and $minute <10) and $registryTime < $currentTime - 600) or
($hour == 6 and ($minute >=0 and $minute <10) and $registryTime < $currentTime - 600) or
($hour == 12 and ($minute >=0 and $minute <10) and $registryTime < $currentTime - 600) or
($hour == 18 and ($minute >=0 and $minute <10) and $registryTime < $currentTime - 600)) {
_log NOTICE, "Time update is now\n";
$registryTime = 0;
}
unless(open REQUEST, $requestFile){
_log ERROR, "Cannot open request file $requestFile\n";
exit 1;
}
$request .= $_ while <REQUEST>;
close REQUEST;
unless(open SIGNATURE, $signatureFile){
_log ERROR, "Cannot open signature file $signatureFile\n";
exit 1;
}
$signature .= $_ while <SIGNATURE>;
close SIGNATURE;
my $soap = SOAP::Lite->service("http://vigruzki.rkn.gov.ru/services/OperatorRequest/?wsdl");
$SOAP::Constants::PREFIX_ENV = 'SOAP-ENV';
# метод возвращает время в милисекундах, нам и секунд достаточно!! )
my $lastUpdateTime = ($soap->getLastDumpDateEx)[1] * 0.001;
# Убрать после теста
_log NOTICE, "File update time: $registryTime\n";
_log NOTICE, "Registry update time: $lastUpdateTime\n";
# Если есть срочные обновления в реестре запращиваем файл реестра и в течении времени ожидания приема файла пробуем загрузить файл раз в step секунд
if($lastUpdateTime > $registryTime){
_log NOTICE, "Trying update registry file\n";
my @sendRequest = $soap->sendRequest($request, $signature);
if($sendRequest[0] eq "true"){
unless(open REGISTRY, ">$registryFile"){
_log ERROR, "Cannot open file $registryFile\n";
exit 1;
}
flock REGISTRY, LOCK_EX;
my $code = $sendRequest[2];
_log NOTICE, "Code for checking\t\t$code\n";
while ($retries > 0) {
_log NOTICE, "Trying get result for check code $code. Wait $retries seconds\n";
$retries -= $step;
sleep $step;
my @getResult = $soap->getResult($code);
if($getResult[0] eq "true"){
_log DEBUG, "Update request for code $code is answered\n";
print REGISTRY decode_base64($getResult[1]);
_log NOTICE, "Registry updated\n";
my $xml = unzip($registryFile);
my ($urls, $domains, $ips, $blockDomains) = parse_xml(unzip($registryFile));
_log DEBUG, "Update ipset";
_ipset $ips;
unless(open SQUIDACL, ">$aclDomainFile"){
_log ERROR, "Cannot open acl file $aclDomainFile\n";
exit 1;
}
flock SQUIDACL, LOCK_EX;
binmode SQUIDACL, ":utf8";
foreach (@{$domains}){
print SQUIDACL "$_\n";
}
close SQUIDACL;
unless(open SQUIDACL, ">$aclUrlFile"){
_log ERROR, "Cannot open acl file $aclUrlFile\n";
exit 1;
}
flock SQUIDACL, LOCK_EX;
binmode SQUIDACL, ":utf8";
foreach (@{$urls}){
print SQUIDACL "$_\n";
}
close SQUIDACL;
unless (open BLOCKHOSTS, ">$hostFile") {
_log ERROR, "Cannot open hosts file $hostFile\n";
exit 1;
}
flock BLOCKHOSTS, LOCK_EX;
binmode BLOCKHOSTS, ":utf8";
foreach (@$blockDomains){
print BLOCKHOSTS "192.168.202.1\t$_\n";
}
close BLOCKHOSTS;
$update = 0;
last;
}
}
close REGISTRY;
}
else{
_log ERROR, "Cannot send request: $sendRequest[1]\n";
}
}
select STDOUT;
close LOGFILE;
exit $update;
