#!/usr/bin/perl # Copyright: 2023 Virtuozzo International GmbH # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # $Id: manage.sh 7933 2012-10-03 13:16:46Z ihor $ #=============================================================================== # # FILE: manage.pl # # USAGE: ./manage.pl -m|--mode mode [-h|--hostname] [-xmx size] [-xms size] [-gc gc] # # DESCRIPTION: This isscript used for managing GlassFish cluster inside Jelastic cloud # # OPTIONS: mode - build, setopts, cleanup, replicationon, replicationoff # REQUIREMENTS: TBD # NOTES: # AUTHOR: $Author: ihor $ # COMPANY: Hivext Technologies Inc. # VERSION: 0.1 # REVISION: $Rev: 7933 $ #=============================================================================== use strict; use warnings; require File::Temp; use Fcntl qw(:flock SEEK_END); use Config::Auto; use File::Temp (); use File::Grep qw( fgrep ); use File::Copy; use Net::SSH::Perl; use Net::IPAddress::Minimal ('invert_ip', 'num_to_ip'); use Socket; use LWP::Simple; use Sys::Hostname; use Data::Dumper; use Log::FileSimple; use Getopt::Long; use Getopt::LongUsage; use IO::File; use constant { JPASS_FILE => '/var/lib/jelastic/glalssfish/gfpass', JPASS_HASH => '/var/lib/jelastic/glalssfish/gfhash', HOST_CONFIG => '/var/lib/jelastic/glalssfish/passwords', HOSTS_FILE => '/etc/hosts', GF_PATH => '/opt/glassfish3', ASADMIN => '/opt/glassfish3/bin/asadmin', VTUN_SERVICE => '/etc/init.d/vtund', VTUND_CONF => '/etc/vtund.conf', DOMAIN_CONF => '/opt/glassfish3/glassfish/domains/domain1/config/domain.xml', ADMIN_KEY_FILE => '/opt/glassfish3/glassfish/domains/domain1/config/admin-keyfile', INSTANCE_PATH => "/opt/glassfish3/glassfish/nodes/", OPT_ASADM => 0, TIMEWAIT => 12, ADMIN_EMAIL => "root", PRINT_MEM => 0, SLEEPSEC => 5 }; my $DAS_PASSWORD_FILE = "/var/lib/jelastic/glalssfish/daspassword"; #my $LOCK_ADMIN_KEYFILE = "/var/lock/glassfish-domian1.keylock"; my $DASPASSWORD = ""; my $JAVA_COMMON_PATH = "/usr/java/"; my $JAVA6_DIR = join '', $JAVA_COMMON_PATH, `ls $JAVA_COMMON_PATH | grep jdk1.6 | head -n 1`; my $JAVA7_DIR = join '', $JAVA_COMMON_PATH, `ls $JAVA_COMMON_PATH | grep jdk1.7 | head -n 1`; my $JAVA8_DIR = join '', $JAVA_COMMON_PATH, `ls $JAVA_COMMON_PATH | grep jdk1.7 | head -n 1`; chomp($JAVA6_DIR); chomp($JAVA7_DIR); chomp($JAVA8_DIR); my $JAVA_LATEST_DIR = "/usr/java/latest"; use constant { ERR_NOERR => 0, ERR_NODENOTFOUND => 1, ERR_NODEEXISTS => 2, ERR_CUSTOM => 99 }; #my $tempadminfile; my $logId=int(rand(1000)); my $logger = new Log::FileSimple( name => "logs-$logId", file => '/var/log/cluster_manager.log', mask => -1, autoflush => 1 ); my $pidFile='/var/run/glmanage.pid'; # check if script is running #sub openPidFile { # my $file=shift; # if(-e $file) { # my $fh=IO::File->new($file) || return; # my $pid=<$fh>; # # kill (0,pid) check for process in process table # die "Script already running with PID $pid" if (kill (0,$pid)); # warn "Removing PID file for defunct script process $pid.\n"; # die "Can't unlink PID file $file" unless -w $file && unlink $file; # } # my $fh=IO::File->new($file,O_WRONLY|O_CREAT|O_EXCL,0644) # or die "Can't create $file: $!\n"; # # write to pid-file current proccess id # print $fh $$; #} sub get_environment_name { my $hostname = shift @_; my @fenvname = split /\./, $hostname; shift @fenvname if $fenvname[0] !~ "glassfish"; return join( ".", @fenvname ); } sub print_environment_name { print get_environment_name hostname(); } sub get_local_ip(){ my $addrcmd = 'ip a l | sed -rne "/tap/{N;d};/inet/{/127.0.0.[0-9]+/d;/inet6/d;s/inet\s+([^\/]+).*/\1/;s/\s*//;/(192\.168(\.[0-9]{1,3}){2})|(172\.(1[6-9]|2[0-9]|3[01])(\.[0-9]{1,3}){2})|(10(\.[0-9]{1,3}){3})|(100\.(6[4-9]|[7-9][0-9]|1[0-1][0-9]|12[0-7])(\.[0-9]{1,3}){2})/p};"|tail -n1'; my $myip = `$addrcmd`; $logger->log( message => "get_local_ip got: $myip"); chomp($myip); $myip =~ s/^\s+//; $logger->log( message => "get_local_ip $myip"); return $myip; } sub get_ext_ip(){ my $addrcmd = 'ip a l | sed -rne "/tap/{N;d};/inet/{/127.0.0.[0-9]+/d;/inet6/d;s/inet\s+([^\/]+).*/\1/;s/\s*//;/(192\.168(\.[0-9]{1,3}){2})|(172\.(1[6-9]|2[0-9]|3[01])(\.[0-9]{1,3}){2})|(10(\.[0-9]{1,3}){3})|(100\.(6[4-9]|[7-9][0-9]|1[0-1][0-9]|12[0-7])(\.[0-9]{1,3}){2})/d;p};"'; my $myip = `$addrcmd`; chomp($myip); $myip =~ s/^\s+//; $logger->log( message => "get_ext_ip $myip"); return $myip; } sub get_das_properties { my $envname = hostname(); my $das = {}; $das->{hostname} = $envname; # $das->{ip} = inet_ntoa( scalar gethostbyname( $envname || 'localhost' ) ); $das->{ip} = get_local_ip(); return $das; } sub build_config { $logger->log( message => "Building nodes hash" ); my $config = Config::Auto::parse( HOST_CONFIG, format => "space" ); my @hosts = map +{ ('ip') => $_ }, keys %$config; my $envname = get_environment_name hostname(); my $das = get_das_properties(); foreach (@hosts) { $$_{password} = $$config{ $$_{ip} }; $$_{id} = invert_ip( $$_{ip} ); if ( $das->{ip} eq $$_{ip} ) { $$_{hostname} = "das.$envname"; $DASPASSWORD = $$_{password}; next; } $$_{hostname} = "node" . $$_{id} . ".$envname"; } $logger->log( message => "\"".`free -m`."\"" ) if ( PRINT_MEM ); $logger->log( message => "Nodes in hash " ); return \@hosts; } sub generate_hosts { my $config = shift @_; my $das = get_das_properties; $logger->log( message => "Generating hosts file for DAS" ); open INFILE, "+>" . HOSTS_FILE or die "Can't open file $!"; my @hostsfile = ; print INFILE "127.0.0.1 localhost localhost.localdomain\n"; my $extIp = &get_ext_ip(); if ( $extIp ne "" ) { #Ext ip defined print INFILE "$extIp $das->{hostname} ${\(hostname())}\n"; }else{ print INFILE "$das->{ip} $das->{hostname} ${\(hostname())}\n"; } $logger->log( message => "config and hosts", objects => [ $config, @hostsfile ] ); foreach (@$config) { unless ( grep { chomp; $_ eq "$$_{ip} $$_{hostname}" } @hostsfile ) { print INFILE "\n$$_{ip} $$_{hostname}\n"; $logger->log( message => "adding record to host file", objects => [$_] ); } } close INFILE; } sub ipchanged(){ my $das = &get_das_properties(); my $localIp = get_local_ip(); my $extIp = &get_ext_ip(); my $ip = $localIp; if ( $extIp ne "" ) { #Ext ip defined $ip = $extIp; } $logger->log( message => "Modifing hosts records localIP: $localIp, exIp: $extIp, IP: $ip" ); # system "sed -i \'/$das->{hostname}\\s*${\(hostname())}/c\\".$ip." $das->{hostname} ${\(hostname())}\' ".HOSTS_FILE; system "sed -i -re \'s/^[0-9\\.]{7,15}\\s*\(${\(hostname())}\)/".$ip." \\1\/g\' ".HOSTS_FILE; if ( ! -f $DAS_PASSWORD_FILE && -f JPASS_FILE && -f JPASS_HASH) { # I'm das my $config = &remove_das( build_config(), &get_das_properties() ); foreach (@$config) { my $ssh = Net::SSH::Perl->new($$_{ip}, privileged => 0); $ssh->login( 'root', $$_{password}); my ($ssh_stdout, $ssh_stderr, $ssh_exit) = $ssh->cmd("/usr/bin/perl /usr/lib/jelastic/libs/glassfish-tools.lib -m ipchanged"); } } } #sub get_das_pas{ # my $das = get_das_properties; # my $config = Config::Auto::parse( HOST_CONFIG, format => "space" ); # my $key; # foreach $key (keys %$config) { # return "$key $$config{$key}" if ( $das->{ip} eq $key ); # } #} sub modify_remote_hostsfile { my $config = shift @_; my $das = get_das_properties; #DEBUG #$logger->log( message => "modify_remote_hostsfile $$config{ip} $$config{password}| $das->{hostname} $das->{ip} | end") ; $logger->log( message => "Modifing hosts records for $$config{hostname} $$config{ip}" ); my $ssh = Net::SSH::Perl->new( $$config{ip}, privileged => 0 ); $ssh->login( 'root', $$config{password} ); my ( $out, undef, undef ) = $ssh->cmd("cat /etc/hosts"); my @res = split( /\n/, $out ); $ssh->cmd("echo '$das->{ip} $das->{hostname}' >> /etc/hosts") unless scalar grep { /${\($das->{ip})}\s+${\($das->{hostname})}/ } @res; $logger->log( message => "$das->{ip} $DASPASSWORD" ); $ssh->cmd("echo \"$das->{ip} $DASPASSWORD\" > $DAS_PASSWORD_FILE"); $ssh->cmd("chkconfig --level 3 glassfish-remote on 2>&1"); # unless scalar grep { /${\($das->{ip})}\s+${\($das->{hostname})}/ } @res; #TODO send das pas to instances } sub get_clusters { my $clusters = &asadmin(" list-clusters"); return split( '\n', `echo '$clusters' | sed '/list-clusters/d;/Nothing/d' | cut -d' ' -f1` ); } sub prepare_cluster { my $config = shift @_; my @commands = ( { command => "create-local-instance --cluster gfcluster", prefix => '', name => 'instance-' }, ); my $das = get_das_properties; my $command; $command=`/etc/init.d/glassfish-domain1 start` unless &get_domain_status("domain1"); # &asadmin("start-domain domain1") unless &get_domain_status("domain1"); my @current_nodes = map { $_ =~ m/instance-(\d+)/ } &get_instances(); unless ( &get_clusters() ) { $logger->log( message => "Preparing DAS" ); my $envname = &get_environment_name( hostname() ); $logger->log( message => "[ ! -d \"/opt/glassfish3/glassfish/nodes/\" ] && { mkdir -p \"/opt/glassfish3/glassfish/nodes/localhost-domain1/instance-".invert_ip($das->{ip})."/logs\";}" ); system("[ ! -d \"/opt/glassfish3/glassfish/nodes/\" ] && { mkdir -p \"/opt/glassfish3/glassfish/nodes/localhost-domain1/instance-".invert_ip($das->{ip})."/logs\";}"); system("/etc/init.d/glassfish-domain1 switch-log"); system("chown glassfish:glassfish -R /opt/glassfish3/glassfish/"); &asadmin("create-cluster gfcluster "); &asadmin("create-local-instance --cluster gfcluster instance-". invert_ip( $das->{ip} )); &asadmin("stop-cluster gfcluster "); &asadmin("start-cluster gfcluster "); } $logger->log( message => "\"".`free -m`."\"" ) if ( PRINT_MEM ); if (&vtundstatus) { &switchgms("true"); } else { &switchgms("false"); } #create ssh-nodes $logger->log( message => "Creating records for SSH nodes on DAS", objects => [$config] ); my $t_config = $config; foreach (@$t_config) { my $test = $_; &add_node($_) unless ( grep /$$test{id}/, @current_nodes ); } $logger->log( message => "Records for SSH nodes on DAS", objects => [$config] ); $logger->log( message => "Current nodes on DAS", objects => [@current_nodes] ); foreach my $node (@current_nodes) { $logger->log( message => "Node: $node" ); next if grep { $$_{id} eq $node } @$config; next if ( invert_ip($node) eq $das->{ip} ); &delete_node($node); } system "chkconfig --level 3 glassfish-domain1 on 2>&1"; $logger->log( message => "DAS configured" ); &start_cluster(); } sub prepare_remote_instances { my $node_config = shift @_; $logger->log( message => "Preparing remote instance-$$node_config{id} $$node_config{hostname} $$node_config{ip}" ); my $das = get_das_properties; my $command = "--host $das->{hostname} --port 4848 create-local-instance --cluster gfcluster instance-$$node_config{id}"; my $user = 'root'; my $envname = &get_environment_name( hostname() ); &asadmin("create-instance --node node$$node_config{id}.$envname --cluster gfcluster instance-$$node_config{id}"); # &asadmin("start-instance instance-$$node_config{id}"); $logger->log( message => "instance-$$node_config{id} ready" ); } sub get_instances { # JE-11378 my @lines = &asadmin("list-instances"); &asadmin("start-domain domain1") if ( scalar( grep { /start-domain/ } @lines )) ; @lines = &asadmin("list-clusters") ; &start_cluster() if ( scalar( grep { /gfcluster not running/ } @lines )); @lines = &asadmin("list-instances"); return split( '\n', `echo '@lines' | sed '/list-instances/d' | cut -d' ' -f1` ); # my $instances = &asadmin("list-instances"); # return split( '\n', `echo '$instances' | sed '/list-instances/d' | cut -d' ' -f1` ); } sub get_nodes_ssh { my $nodes = &asadmin("list-nodes"); return split( '\n', `echo '$nodes' | sed '/list-nodes/d;/CONFIG/d' | cut -d' ' -f1` ); } sub add_node { my $node_config = shift @_; $logger->log( message => "Adding node", objects => [$node_config] ); #check if instance already exists my $node_id = $node_config->{id}; $logger->log( message => "looking for node $node_id" ); my @inst = &get_instances; $logger->log( message => "instances", objects => [@inst] ); if ( grep /$node_id/, @inst ) { $logger->log( message => "Node already exists" ); return ERR_NODEEXISTS; } modify_remote_hostsfile($node_config); &asadmin( "--interactive=false create-password-alias password-$node_config->{id}", "AS_ADMIN_ALIASPASSWORD=$node_config->{password}" ); my $envname = &get_environment_name( hostname() ); my $ssh = Net::SSH::Perl->new( $$node_config{ip}, privileged => 0 ); $ssh->login( 'root', $$node_config{password} ); $ssh->cmd("[ ! -d \"/opt/glassfish3/glassfish/nodes/\" ] && { mkdir -p \"/opt/glassfish3/glassfish/nodes/node$node_config->{id}.$envname/instance-$node_config->{id}/logs\"; }"); $logger->log( message => "[ ! -d \"/opt/glassfish3/glassfish/nodes/\" ] && { mkdir -p \"/opt/glassfish3/glassfish/nodes/node$node_config->{id}.$envname/instance-$node_config->{id}/logs\"; }"); $ssh->cmd("/etc/init.d/glassfish-domain1 switch-log"); $ssh->cmd("chown glassfish:glassfish -R /opt/glassfish3/glassfish/"); &asadmin( "create-node-ssh --sshuser=root --nodehost $node_config->{ip} node$node_config->{id}.$envname", "AS_ADMIN_SSHPASSWORD=\\\$\\{ALIAS=password-$node_config->{id}\\}" ); prepare_remote_instances($node_config); return ERR_NOERR; } sub delete_node { my $node_id = shift @_; return ERR_NODENOTFOUND unless ( grep /$node_id/, &get_instances ); &asadmin("stop-instance instance-$node_id"); &asadmin("delete-instance instance-$node_id"); return ERR_NODENOTFOUND unless ( grep /$node_id/, &get_nodes_ssh ); my $envname = &get_environment_name( hostname() ); &asadmin("delete-node-ssh node$node_id.$envname"); &asadmin("delete-password-alias password-$node_id"); return ERR_NOERR; } sub start_cluster { $logger->log( message => "starting cluster" ); &asadmin("start-cluster gfcluster"); } sub get_domain_status { my $domain = shift @_; my $list = `${\(ASADMIN)} list-domains | grep $domain`; $list = ( $list =~ m/not\s+running/ ? 0 : 1 ) ; $logger->log( message => "Domain status $list" ); return $list ; } sub cleanup_system { # TODO check # /etc/hosts change ip on instances my $config = shift @_; replication_off(); $logger->log( message => "starting cleanup" ); my @ids_hash = (); my %nodes_names = (); my $fl ; my $key; my $domain_xml; my $envname = get_environment_name hostname(); my $hostname;my $cur_node;my $das_old_id; foreach $cur_node (@$config) { $logger->log( message => "$$cur_node{ip}, $$cur_node{id} " ); my $ssh = Net::SSH::Perl->new($$cur_node{ip}, privileged => 0); $ssh->login( 'root', $$cur_node{password}); my ($ssh_stdout, $ssh_stderr, $ssh_exit) = $ssh->cmd("ls ".INSTANCE_PATH." | sed -ne 's/.*node\\([[:digit:]]\\{6,10\\}\\).*/\\1/p;'"); chomp($ssh_stdout); push @ids_hash , $ssh_stdout ; # old node_id push @ids_hash , $$cur_node{id} ; # new node_id #### &asadmin( "delete-password-alias password-$ssh_stdout"); ($ssh_stdout, $ssh_stderr, $ssh_exit) = $ssh->cmd("/usr/bin/perl /usr/lib/jelastic/libs/glassfish-tools.lib -m getenvname"); # chomp($ssh_stdout); $nodes_names{$$cur_node{id}} = $ssh_stdout ; } system "rm -f /opt/glassfish3/glassfish/domains/domain1/config/domain-passwords"; # get das ids opendir(my $dh, INSTANCE_PATH."localhost-domain1/") || die "Cant open ".INSTANCE_PATH . " $!" ; while($fl = readdir ($dh)) { if ( $fl =~ /instance-([0-9]{6,10})/ ) { $das_old_id = $1; last; } } my $das_new_id = invert_ip( get_local_ip() ); push @ids_hash, $das_old_id; push @ids_hash, $das_new_id; $nodes_names{$das_new_id} = get_environment_name hostname() ; $logger ->log( message => "ids hash ". @ids_hash ); my $ids_string = join ("-", @ids_hash); foreach (@$config) { modify_remote_hostsfile ($_); my $ssh = Net::SSH::Perl->new($$_{ip}, privileged => 0); $ssh->login( 'root', $$_{password}); $ssh->cmd("/usr/bin/perl /usr/lib/jelastic/libs/glassfish-tools.lib -m cleanup-remote-instance --idshash ". $ids_string ); } system "rm -rf ".INSTANCE_PATH."/localhost-domain1/instance-".$das_old_id."/logs/instance-".$das_old_id."/"; my @domains_files= ( INSTANCE_PATH."localhost-domain1/".$fl."/config/domain.xml" , "/opt/glassfish3/glassfish/domains/domain1/config/domain.xml" ) ; my %ids_hash = @ids_hash; foreach $domain_xml (@domains_files) { foreach $key (keys %ids_hash){ my $newid = $ids_hash{$key} ; my $newip = num_to_ip($newid) ; my $old_ip = num_to_ip($key) ; $hostname = "node" . $newid . ".".$nodes_names{$newid}; $logger->log( message => "In cleanup. conf: $domain_xml. newid: $newid, New IP: $newip, Hostname : $hostname, Old nodeId: $key, Old IP : $old_ip "); system "sed -i 's/node$key\\.glassfish[[:digit:]\\.a-z\\-]\\{1,\\}/$hostname/' $domain_xml"; system "sed -i 's/instance-$key/instance-$newid/' $domain_xml"; system "sed -i 's/password-$key/password-$newid/' $domain_xml"; system "sed -i 's/$old_ip/$newip/' $domain_xml"; } } $logger->log( message => "\"".`free -m`."\"" ) if ( PRINT_MEM ); $logger->log (message =>"Rename " . INSTANCE_PATH."localhost-domain1/instance-$das_old_id , ".INSTANCE_PATH."localhost-domain1/instance-$das_new_id") ; my $renameres = rename ( INSTANCE_PATH."localhost-domain1/instance-".$das_old_id, INSTANCE_PATH."localhost-domain1/instance-".$das_new_id) ; if ( ! $renameres ){ $logger->log (message =>"Rename failed") ; if ( -d INSTANCE_PATH."localhost-domain1/instance-".$das_new_id ){ system "rm -rf ".INSTANCE_PATH."localhost-domain1/instance-".$das_old_id; } } $logger->log( message => "shuting down cluster" ); foreach $cur_node (@$config) { &asadmin( "stop-instance --force=true instance-$$cur_node{id}" ); } &asadmin( "stop-domain --force=true domain1" ); system "rm -rf /opt/glassfish3/glassfish/domains/domain1/config/.instancestate"; &asadmin( "start-domain domain1"); foreach $cur_node (@$config) { &asadmin("--interactive=false create-password-alias password-$$cur_node{id}","AS_ADMIN_ALIASPASSWORD=$$cur_node{password}" ); } system ("/etc/init.d/glassfish-domain1 restart"); my $line = &asadmin("list-instances"); my @lines = split /\n/, $line; @lines = grep ("instance", @lines); foreach (@lines){ $logger->log(message => "line: $_"); if (/not running/){ my @res = split ; if ( $res[0] =~ /instance/){ $logger->log(message => "$res[0] stoped. Try to start."); &asadmin( "stop-instance --force=true --kill=true $res[0]"); system (" /usr/bin/perl /usr/lib/jelastic/libs/glassfish-tools.lib -m start-remote-instance --instname=". $res[0] ." &"); } } } $logger->log ( message => "exit from cleanup_system " ); } sub cleanup_remote_instance{ &asadmin( "stop-local-instance"); replication_off(); #my $new_ip =#inet_ntoa( scalar gethostbyname( hostname() || 'localhost' ) ); my $new_ip = get_local_ip(); my $node_id = invert_ip( $new_ip ); my $envname = get_environment_name hostname(); my $old_ip; my $dir ; my $old_node_id; my $hostname ; my $ids_string = shift @_; my $key ; $logger->log ( message => "Cleanup remote instance" ); my %ids_hash = split /-/,$ids_string; opendir(my $dh, INSTANCE_PATH ) || die "Cant open dir $!"; while($dir = readdir( $dh)) { if ($dir =~ /node([0-9]{6,10})\./ ){ $old_node_id = $1; $old_ip= num_to_ip($old_node_id); last; } } closedir $dh; if ( ! $old_ip ) { $logger->log( message => "Error getting old IP" ) ; exit; } my $domain_xml= INSTANCE_PATH.$dir."/instance-".$old_node_id."/config/domain.xml"; # TODO: Need to refactor this ugly hack $logger->log( message => "rm -rf ".INSTANCE_PATH."$dir/instance-$old_node_id/logs/instance-$old_node_id/"); system "rm -rf ".INSTANCE_PATH.$dir."/instance-".$old_node_id."/logs/instance-".$old_node_id."/"; # key old node_id, value -new node id foreach $key (keys %ids_hash){ $logger->log ( message => "key $key value $ids_hash{$key}" ); } foreach $key (keys %ids_hash){ my $newid = $ids_hash{$key} ; my $newip = num_to_ip($newid); $hostname = "node" . $newid . ".$envname"; $logger->log( message => "In cleanup remote. Newid: $newid, New IP: $newip, Hostname : $hostname, Old nodeId: $key, Old IP : $old_ip"); system "sed -i 's/node$key\\.glassfish[[:digit:]\\.a-z\\-]\\{1,\\}/$hostname/' $domain_xml"; system "sed -i 's/instance-$key/instance-$newid/' $domain_xml"; system "sed -i 's/password-$key/password-$newid/' $domain_xml"; system "sed -i 's/$old_ip/$newip/' $domain_xml"; } $hostname = "node" . $node_id . ".$envname"; #TODO check rename status rename ( INSTANCE_PATH.$dir , INSTANCE_PATH.$hostname ); rename ( INSTANCE_PATH.$hostname."/instance-".$old_node_id, INSTANCE_PATH.$hostname."/instance-".$node_id ); my $config = Config::Auto::parse( HOSTS_FILE, format => "space" ); $$_{password} = $$config{ $$_{ip} }; my $das_config = Config::Auto::parse( $DAS_PASSWORD_FILE, format => "space" ); foreach my $das_ip ( keys %$das_config ) { system "sed -in '/agent.das.host/cagent.das.host=$$config{$das_ip}' ".INSTANCE_PATH."$hostname/agent/config/das.properties"; $logger->log ( message => "sed -in '/agent.das.host/cagent.das.host=$$config{$das_ip}' ".INSTANCE_PATH."$hostname/agent/config/das.properties"); } system "chown -R glassfish:glassfish /opt/glassfish3/glassfish/*"; $logger->log ( message => "exit from cleanup_remote_instance " ); } sub set_options { my $options = shift @_; $logger->log( message => "setting options" ); if (OPT_ASADM) { &asadmin("start-domain domain1") if get_domain_status("domain1"); my @jvmopts = &asadmin("list-jvm-options"); &asadmin("delete-jvm-options -- ${\(grep {/Xmx/i } @jvmopts)}") if ( scalar( grep { /Xmx/i } @jvmopts ) && defined( $$options{xmx} ) ); &asadmin("delete-jvm-options -- ${\(grep {/Xms/ } @jvmopts)}") if ( scalar( grep { /Xms/ } @jvmopts ) && defined( $$options{xms} ) ); &asadmin("create-jvm-options -- -Xmx$$options{xmx}") if ( defined( $$options{xmx} ) ); &asadmin("create-jvm-options -- -Xms$$options{xms}") if ( defined( $$options{xms} ) ); ## GC manipulation my $currentGC = join '\n', grep { /Use(Parallel|G1|ConcMarkSweep|Serial)GC/ } @jvmopts; $currentGC =~ s/\n//g; $currentGC =~ s/:\s/\\:\+/g; &asadmin("delete-jvm-options \'$currentGC\'") unless ( $currentGC eq '' ); $$options{gc} =~ s/:/\\:/g; &asadmin("create-jvm-options -- '$$options{gc}'") if ( defined( $$options{gc} ) ); &asadmin("stop-domain domain1"); } else { open IN, "+<" . DOMAIN_CONF; my @lines = ; open IN, ">" . DOMAIN_CONF; foreach (@lines) { $_ =~ s/-Xmx(\d+)[Mm]/-Xmx${\($$options{xmx})}/gi; # $_ =~ s/-Xms(\d+)[Mm]/-Xms${\($$options{xms})}/gi; $_ =~ s/-XX:\+Use(Parallel|G1|ConcMarkSweep|Serial)GC/$$options{gc}/gi; print IN $_; } close IN; } $logger->log( message => "options are set" ); system "chkconfig --level 3 glassfish-domain1 on 2>&1"; } sub replication_on { my $options = shift @_; my $config = shift @_; $logger->log( message => "stoping ${\(VTUN_SERVICE)} " ); system "${\(VTUN_SERVICE)} stop"; #clear mac system "sed -i '/^ip \"link/d' ${\(VTUND_CONF)}" if fgrep { /ip\s+"link/ } "${\(VTUND_CONF)}"; system "sed -i 's/REMOTE_HOST[\=[:digit:]\.]\\{1,\\}/REMOTE_HOST=/' ${\(VTUN_SERVICE)}"; #sethost system "sed -i '/^REMOTE_HOST=/cREMOTE_HOST=$$options{hostname}' ${\(VTUN_SERVICE)}"; chmod 0777, ${ \(VTUN_SERVICE) }; $logger->log( message => "starting ${\(VTUN_SERVICE)} " ); system "${\(VTUN_SERVICE)} start"; system "chkconfig --level 3 vtund on 2>&1"; my $mac = ''; while ( $mac eq '' ) { sleep 1; my $mac_addr = 'ip l show tap | sed -n -re "/\s*link/{s/.*ether\s+(\S+)\s+.*/\1/;p}"'; $mac = `$mac_addr`; } chomp($mac); my $command = 'sed -i "/^ifconfig/iip \\"link set %% addr ' . $mac . '\\";"' . " ${\(VTUND_CONF)}"; $logger->log( message => "$command" ); system $command; my $list = `${\(ASADMIN)} restart-local-instance 2>/dev/null`; $logger->log( message => "Restart local instance $list"); # wait tap interface receive IP my $tap_ip=''; while ( $tap_ip eq '' ) { sleep 1; if ( `ip a show tap 2>/dev/null` =~ /inet ([\d.]+)/){ $tap_ip = $1; } } # unless ( $$options{remote} ) { # $logger->log( message => "modifing remote hosts" ); # my $das = get_das_properties; # foreach (@$config) { # my $ssh = Net::SSH::Perl->new( $$_{ip} , privileged => 0); # $logger->log( message => "processing $$_{ip}" ); # $ssh->login( 'root', $$_{password} ); # $ssh->cmd("/root/scripts/manage.sh -m replicationon --hostname $$options{hostname} --remote=1"); # } # } } sub replication_off { my $options = shift @_; my $config = shift @_; $logger->log( message => "stoping ${\(VTUN_SERVICE)} " ); system "${\(VTUN_SERVICE)} stop"; system "chkconfig vtund off 2>&1"; system "sed -i '/^ip \"link/d' ${\(VTUND_CONF)}" if fgrep { /ip\s+"link/ } "${\(VTUND_CONF)}"; system "sed -i 's/REMOTE_HOST[\=[:digit:]\.]\\{1,\\}/REMOTE_HOST=/' ${\(VTUN_SERVICE)}"; # unless ( $$options{remote} ) { # $logger->log( message => "modifing remote hosts" ); # my $das = get_das_properties; # foreach (@$config) { # my $ssh = Net::SSH::Perl->new( $$_{ip} , privileged => 0); # $logger->log( message => "processing $$_{ip}" ); # $ssh->login( 'root', $$_{password} ); # $ssh->cmd("/root/scripts/manage.sh -m replicationoff --remote=1"); # } # } } sub deploy { my $options = shift @_; my $filename = "/tmp/$$options{context}.$$options{ext}"; $logger->log( message => "deploing ${\($$options{context})}.${\($$options{ext})}" ); # print "Saving $$options{url} to $filename ..."; # my $res = getstore( $$options{url}, $filename ); &check_jelastic_user(); unless ( &get_domain_status("domain1")) { &asadmin("start-domain domain1"); } my @lines = &asadmin("list-clusters"); &start_cluster() if ( scalar( grep { /gfcluster not running/ } @lines )); my $ua = LWP::UserAgent->new(); if ( $$options{url} =~ /^https/ ){ $ua->ssl_opts( verify_hostname => 0); $ua->ssl_opts( SSL_verify_mode => 0); } $ua->env_proxy; my $request = HTTP::Request->new(GET => $$options{url}); my $response = $ua->request( $request, $filename ); $logger->log( message => "Response:". $response->as_string ); if ( $response->code != 200 || $response->{_headers}->{'content-length'} == 0) { $logger->log( message => "Error download file! Response: ". $response->code ); return -100; } if ( defined ( $response->{_headers}->{'content-disposition'} ) && $response->{_headers}->{'content-disposition'} =~ /ear\"$/){ rename $filename, "/tmp/$$options{context}.ear"; $filename = "/tmp/$$options{context}.ear"; } my $context_opt = ""; $context_opt = "--contextroot '/'" if $$options{context} eq 'ROOT'; $logger->log( message => "Deploying $context_opt $filename" ); my $res = &asadmin("deploy --availabilityenabled=true --asyncreplication=true --force=true --target gfcluster $context_opt $filename" ); if ( $res =~ /failed/ ){ $logger->log(message => "Deploy failed"); system("echo \"$res\" >> /var/log/actions.log"); return -200; } open INFILE, ">/tmp/$$options{context}.url"; print INFILE "$$options{url}"; close INFILE; return 0; } sub undeploy { my $options = shift @_; &check_jelastic_user(); unless ( &get_domain_status("domain1")) { &asadmin("start-domain domain1"); } my @lines = &asadmin("list-clusters"); &start_cluster() if ( scalar( grep { /gfcluster not running/ } @lines )); $logger->log( message => "undeploing ${\($$options{context})}" ); &asadmin("undeploy --target gfcluster $$options{context}"); } sub migrate { my $config = shift @_; foreach (@$config) { my $ssh = Net::SSH::Perl->new( $$config{ip}, privileged => 0); $ssh->login( 'root', $$config{password} ); $ssh->cmd("chkconfig --level 3 glassfish-remote on 2>&1"); } } sub start_local_instance { my $node_id = invert_ip( inet_ntoa( scalar gethostbyname( hostname() || 'localhost' ) ) ); my $envname = get_environment_name hostname(); &check_jelastic_user(); &asadmin("start-local-instance --node node$node_id.$envname instance-$node_id"); # move( $fh->filename(), "${\(GF_PATH)}/glassfish/nodes/node$node_id.$envname/instance-$node_id/config/admin-keyfile" ); } sub remove_das { my $config = shift @_; my $ip = shift @_; my @res = (); foreach (@$config) { next if $$_{ip} eq $$ip{ip}; push @res, $_; } return \@res; } sub start_remote_instance { $logger->log( message => "in start_remote_instance" ); my $r_instance = shift @_; &check_jelastic_user(); my $config = &remove_das( build_config(), &get_das_properties() ); my $password = &get_jelastic_password; my $checkgfcluster ; my $start_executed = 0; $logger->log( message => "start_remote_instance called to start $r_instance" ); my @lines; my $inst_ip; my $inst_pass;my $inst_hostname; foreach my $cur_node (@$config) { if ( $r_instance =~ $$cur_node{id}){ $inst_ip = $$cur_node{ip}; $inst_pass = $$cur_node{password}; $inst_hostname = $$cur_node{hostname}; } } $logger->log(message => "$inst_ip $inst_pass $inst_hostname" ); while ( $start_executed == 0 ) { @lines = &asadmin("list-instances"); &asadmin("start-domain domain1") if ( scalar( grep { /start-domain/ } @lines )); @lines = &asadmin("list-clusters"); &start_cluster() if ( scalar( grep { /gfcluster not running/ } @lines )); @lines = &asadmin("list-instances"); $logger->log(message => "check instance" ); @lines = grep {$r_instance} @lines; foreach (@lines) { if (/not running/) { $logger->log(message => "starting $r_instance"); my $res = &asadmin("start-instance $r_instance"); if ( $res =~ /failed/ ){ $logger->log(message => "start failed"); my @chars = ("A".."Z", "a".."z"); my $string; $string .= $chars[rand @chars] for 1..8; my $ssh = Net::SSH::Perl->new( $inst_ip, privileged => 0 ); $ssh->login( 'root', $inst_pass ); $ssh->cmd("echo AS_ADMIN_PASSWORD=$password>/tmp/$string"); $ssh->cmd("/opt/glassfish3/bin/asadmin start-local-instance --node $inst_hostname --sync normal $r_instance -u jelastic -W /tmp/$string"); } $start_executed = 1; }else { $start_executed = 1; } } } $logger->log( message => "out start_remote_instance" ); } sub vtundstatus { my $status = `chkconfig --list vtund 2>&1 | grep -Po "3:(on|off)" | grep -Po "(on|off)" | tr -d '\n'`; return 1 if ( $status eq "on" ); return 0; } sub switchgms { my $mode = shift @_; open IN, "+<" . DOMAIN_CONF; my @lines = ; seek IN, 0, 0; foreach (@lines) { if ( $_ =~ /gms-enabled="(true|false)"/ ) { $_ =~ s/gms-enabled="(true|false)"/gms-enabled="$mode"/gi if ( $mode ne $1 ); print IN $_; last; } $_ =~ s/name="gfcluster"/name="gfcluster" gms-enabled="$mode"/gi; print IN $_; } close IN; return 1; } sub start_instance { # called on instance if ( !-f $DAS_PASSWORD_FILE ) { $logger->log( message => "Error: No file with das ip/password" ); return; #TODO wait, exit or start in some minutes? } #my $interface = "venet0:0"; #my $ifconfig = "/sbin/ifconfig"; #my $myip = 0; #my @lines = qx|$ifconfig $interface| or die( "Can't get info from ifconfig: " . $! ); #foreach (@lines) { # if (/inet addr:([\d.]+)/) { # $myip = "$1\n"; # chomp($myip); # } #} my $myip_local_ip = get_local_ip(); my $instance = "instance-" . invert_ip($myip_local_ip); my $config = Config::Auto::parse( $DAS_PASSWORD_FILE, format => "space" ); my $dasip; my $daspas; foreach my $a ( keys %$config ) { $daspas = $$config{$a}; $dasip = $a; } # print "My instance name: $instance, das ip: $dasip, daspas: $daspas"; $logger->log( message => "My instance name: $instance, das ip: $dasip" ); my $ssh = Net::SSH::Perl->new($dasip, privileged => 0); $ssh->login( 'root', $daspas ); # print "/root/scripts/manage.sh -remote -m start-remote-instance --instname=$instance"; $ssh->cmd(" /usr/bin/perl /usr/lib/jelastic/libs/glassfish-tools.lib -m start-remote-instance --instname=$instance"); } sub stop_instance { $logger->log( message => "stop-instance" ); } sub generate_random_password { my $password; my $_rand; my $password_length = $_[0]; if ( !$password_length ) { $password_length = 10; } my @chars = split( " ", "a b c d e f g h i j k l m n o p q r s t u v w x y z - _ % # | 0 1 2 3 4 5 6 7 8 9" ); srand; for ( my $i = 0 ; $i <= $password_length ; $i++ ) { $_rand = int( rand 41 ); $password .= $chars[$_rand]; } return $password; } sub check_jelastic_user { my $hasuser=0; open PSWD, "+<" . ADMIN_KEY_FILE; my @users=; foreach my $user (@users) { if ( $user =~ m/^jelastic;(.*);asadmin/ ) { open ETHASH, "<" . JPASS_HASH or last; while () { $hasuser++ unless($1 =~ /\Q$_\E/); } close ETHASH; last; } } &add_jelastic_user unless ($hasuser); close PSWD; } sub add_jelastic_user { my $result; my $hasuser=0; open PSWD, "+<" . ADMIN_KEY_FILE; my @users=; #check if we already have user jelastic $hasuser++ if grep { $_ =~ m/jelastic;.*;asadmin/ } @users; @users = grep { $_ !~ m/jelastic;.*;asadmin/ } @users; push @users, "jelastic;{SSHA256}DdW1VfFCD0AqbQFzsu6Swqel1g1gZZ6f1m87JX6FQYSpu1X/BxTX5A==;asadmin"; #rewrite admin keyfile flock PSWD, 2; seek PSWD, 0, 0; print PSWD @users; print PSWD "\n"; close PSWD; unless ($hasuser) { $result = `/etc/init.d/glassfish-domain1 stop`; $logger->log( message => "New user added. Stopping domain", objects => [$result] ); $result = `/etc/init.d/glassfish-domain1 start`; $logger->log( message => "New user added. Starting domain", objects => [$result] ); } my $pwdfile = File::Temp->new(); chmod 0666, $pwdfile; my $password = &generate_random_password; $logger->log( message => "saving password" ); open JPWD, ">" . JPASS_FILE; print JPWD $password; close JPWD; $logger->log( message => "AS_ADMIN_PASSWORD=\nAS_ADMIN_NEWPASSWORD=*****" ); print $pwdfile "AS_ADMIN_PASSWORD=\nAS_ADMIN_NEWPASSWORD=$password"; $pwdfile->seek( 0, SEEK_END ); $result = `sudo -u glassfish ${\(ASADMIN)} -u jelastic -W ${\($pwdfile->filename())} change-admin-password`; $logger->log( message => "sudo -u glassfish ${\(ASADMIN)} -u jelastic -W ${\($pwdfile->filename())} change-admin-password", objects => [$result] ); #get etalon values open PSWD, "<" . ADMIN_KEY_FILE; while () { if ( $_ =~ /^jelastic;(.*);asadmin/ ) { open JPSWD, ">" . JPASS_HASH; print JPSWD $1; close JPSWD; last; } } close PSWD; } sub get_jelastic_password { open PSWD, "<" . JPASS_FILE; chomp(my $password = ); close PSWD; return $password; } sub asadmin { my $params = shift @_; my $addpass = shift @_ || ''; my $result; my $command = "sudo -u glassfish ${\(ASADMIN)} -u jelastic "; my $password = &get_jelastic_password; my $pwdfile = File::Temp->new(); chmod 0666, $pwdfile; print $pwdfile "AS_ADMIN_PASSWORD=$password\n$addpass"; $pwdfile->seek( 0, SEEK_END ); $command = "$command -W ${\($pwdfile->filename())} $params"; $result = `$command 2>&1`; $logger->log( message => "$command", objects => [$result] ); return $result; } sub setengine { my $engine = shift @_ || 'java6'; my $java_dir=""; if($engine =~ /^java6/) { $java_dir = $JAVA6_DIR; } elsif ($engine =~ /^java7/) { $java_dir = $JAVA7_DIR; } elsif ($engine =~ /^java8/) { $java_dir = $JAVA8_DIR; } else { print "Incorrect Java Type"; return; } if ( -l "$JAVA_LATEST_DIR" ) { unlink "$JAVA_LATEST_DIR" or die "Failed to remove file $JAVA_LATEST_DIR: $!\n"; } symlink($java_dir, $JAVA_LATEST_DIR) or die print "$!\n"; } ################# PARSE OPTIONS ################################ #openPidFile($pidFile); my $actions = {}; my @args; $actions->{remote} = 0; #start_instance; my @getoptconf = ( "h|hostname:s" => \$actions->{hostname}, "mode|m=s" => \$actions->{mode}, "xmx:s" => \$actions->{xmx}, "xms:s" => \$actions->{xms}, "gc:s" => \$actions->{gc}, "warpath|p:s" => \$actions->{url}, "context|c:s" => \$actions->{context}, "ext|e:s" => \$actions->{ext}, "remote|r:i" => \$actions->{remote}, "instname:s" => \$actions->{instname}, "idshash:s" => \$actions->{idshash}, '<>' => sub { push (@args, $_[0]->{name})}, ); my $usage = sub { my @getopt_long_configuration = @_; GetLongUsage( 'cli_use' => ( $0 . " [options]" ), 'descriptions' => [ 'mode' => "mode - build, setopts, cleanup, replicationon, replicationoff, start-local-instance, start-remote-instance, start, stop, setengine", 'hostname' => "the hostname of balancer", 'xmx' => "Xmx opt", 'xms' => "Xms opt", 'gc' => "gc" ], 'Getopt_Long' => \@getopt_long_configuration, ); }; GetOptions(@getoptconf) || die( $usage->(@getoptconf), "\n" ); $logger->log( message => "Start ", objects => [$actions] ); $logger->log( message => "\"".`free -m`."\"" ) if ( PRINT_MEM ); $logger->log( message => "Count ".` ps aux | grep -P "glassfish-[domain1|remote] start" | grep -v grep `); if ( $actions->{mode} eq 'generate-hosts' ) { if ( -f HOST_CONFIG) { my $config = &remove_das( build_config(), &get_das_properties() ); generate_hosts($config); }else{ $logger->log( message => "No file", HOST_CONFIG ); } $logger->log( message => "Exit from script"); $logger = undef; exit 0; } if ( $actions->{mode} ne 'cleanup' ) { my $count = ` ps aux | grep -v grep | grep -c -P "glassfish-[domain1|remote] start|start-remote-instance"`; my $waitLimit=150; while ( $count != 0 && $waitLimit > 0 ) { sleep 2; $count = `ps aux | grep -v grep | grep -c -P "glassfish-[domain1|remote] start|start-remote-instance"`; $logger->log( message => "count = $count; waitlimit= $waitLimit "); $waitLimit--; } }else{ system "killall java 1>/dev/null 2>&1"; system "killall -9 java 1>/dev/null 2>&1"; } ############## PARSE DONE #################################### if ( ! $actions->{mode} ) { exit 0; } if ( $actions->{mode} eq 'build' ) { my $config = &remove_das( build_config(), &get_das_properties() ); &check_jelastic_user(); generate_hosts($config); prepare_cluster($config); chmod 0666, DOMAIN_CONF; $logger->log( message => "Finish build"); } if ( $actions->{mode} eq 'cleanup' ) { my $config = &remove_das( build_config(), &get_das_properties() ); cleanup_system($config); $logger->log( message => "Finish cleanup"); # TODO check /etc/hosts file #generate_hosts($config); } if ( $actions->{mode} eq 'setopts' ) { set_options($actions); chmod 0666, DOMAIN_CONF; $logger->log( message => "Finish setopts"); } if ( $actions->{mode} eq 'migrate' ) { my $config = &remove_das( build_config(), &get_das_properties() ); &migrate($config); $logger->log( message => "Finish migrate"); } if ( $actions->{mode} eq 'setengine' ) { &setengine($args[0]); $logger->log( message => "Finish setengine"); } if ( $actions->{mode} eq 'start-local-instance' ) { &start_local_instance(); $logger->log( message => "Finish start-remote-instance"); } if ( $actions->{mode} eq 'replicationon' ) { my $config; die "Please specify hostname" unless defined $actions->{hostname}; $config = build_config() if ( $actions->{remote} ); replication_on( $actions, $config ); $logger->log( message => "Finish replicationon"); } if ( $actions->{mode} eq 'replicationoff' ) { my $config; $config = build_config() if ( $actions->{remote} ); replication_off( $actions, $config ); $logger->log( message => "Finish replicationoff"); } if ( $actions->{mode} eq 'ipchanged' ) { &ipchanged(); $logger->log( message => "Finish ipchanged"); } if ( $actions->{mode} eq 'deploy' ) { &check_jelastic_user(); my $deploy_result = deploy($actions) ; if ( $deploy_result != 0 ) { $logger->log( message => "Deploy error"); $logger = undef; exit ($deploy_result); }else { $logger->log( message => "Finish deploy"); } } if ( $actions->{mode} eq 'undeploy' ) { &check_jelastic_user(); undeploy($actions); $logger->log( message => "Finish undeploy"); } if ( $actions->{mode} eq 'start-remote-instance' ) { # my $config = &remove_das( build_config(), &get_das_properties() ); die "Please specify instance" unless defined $actions->{instname}; start_remote_instance( $actions->{instname} ); $logger->log( message => "Finish start-remote-instance"); } if ( $actions->{mode} eq 'cleanup-remote-instance' ) { # idshash die "Please specify ids " unless defined $actions->{idshash}; cleanup_remote_instance( $actions->{idshash} ); $logger->log( message => "Finish cleanup-remote-instance"); } if ( $actions->{mode} eq 'start' ) { start_instance(); $logger->log( message => "Finish start instance"); } if ( $actions->{mode} eq 'stop' ) { stop_instance(); $logger->log( message => "Finish stop instance"); } if ( $actions->{mode} eq 'getenvname' ) { print_environment_name(); $logger->log( message => "Finish getenvname"); } if ( $actions->{mode} eq 'createadminuser' ) { &check_jelastic_user(); $logger->log( message => "Admin user created"); } $logger->log( message => "\"".`free -m`."\"" ) if ( PRINT_MEM ); $logger->log( message => "Exit from script"); $logger = undef; # exit 0;