#! /usr/bin/perl -w use strict; my ($fh, $dh); my %parts; my (@seq, $pair, $data, $section, $name); # $dir => # section => '...' # seq => ( # { reg=> ..., data=> ..., op=>} // pair # ) my (@list); my %projects = ( 'emi3' => 'EMI-3', 'glite' => 'gLite', 'native' => 'native', 'jenkins' => 'jenkins', 'egi' => 'EGI', 'umd' => 'UMD', 'upstream' => 'Upstream', ); my %platforms = ( 'sl6' => 'sl6', 'sl6-32' => 'sl6-32', 'sl6-64' => 'sl6', 'sl7-64' => 'sl7', 'deb7' => 'deb7', 'deb7-64' => 'deb7', 'deb8' => 'deb8', 'deb8-64' => 'deb8', 'deb9' => 'deb9', 'deb9-64' => 'deb9', 'deb10' => 'deb10', 'deb10-64' => 'deb10', 'f19' => 'f19', 'f19-32' => 'f19-32', 'f19-64' => 'f19', 'f20' => 'f20', 'f20-32' => 'f20-32', 'f20-64' => 'f20', 'fedora' => 'f', 'ubnt14-64' => 'ubnt14', 'ubnt16-64' => 'ubnt16', ); my %scenarios = ( 'install' => '', 'minor' => 'upgrade-minor-', 'major' => 'upgrade-major-', 'double' => 'upgrade-double-', 'eticsless' => '', 'perftest' => '', 'upgrade' => 'upgrade-', ); # repository type into category name my %categories = ( 'prod' => 'PRODUCTION', 'eticsless' => 'NIGHTLY', 'perftest' => 'NIGHTLY', 'release' => 'RELEASE', 'testing' => 'TESTING', 'latest' => 'LATEST', 'emi3-branch' => 'EMI3-BRANCH', ); # ==== load ==== opendir $dh, "." || die "Can't open '.': $!"; for my $dir (readdir $dh) { if ($dir=~/^deploy([0-9]*)-(.*)/) { $parts{$dir}{section}=$2; } } closedir $dh; for my $file (keys %parts) { @seq = (); open $fh, '<', $file || die "Can't open '$file': $!"; while (<$fh>) { chomp; while ((not $_) and ($_=<$fh>)) { chomp; } if (not $_) { last; } #print "$file: line: $_\n"; $pair = {}; $pair->{reg} = qr/.*$_.*/; #print "$file: regex: $_\n"; $data = ""; while (<$fh>) { chomp; if (/^(\+|=|\.)$/) { $pair->{op}=$1; #print "$file: op: $1\n"; last; } $data = "$data$_\n"; } #print "$file: data: ".length($data)." bytes\n"; $pair->{data} = $data; #print "$file: sequence added\n"; push @seq, $pair; } close $fh; $parts{$file}{seq} = [ @seq ]; } open $fh, '<', 'list' || die "Can't open 'list': $!"; while (<$fh>) { my $name; chomp; next if /^$/; next if /^#/; push @list, $_; } close $fh; # ==== print ==== for my $part (sort keys %parts) { print "$parts{$part}{section}:\n"; @seq = @{$parts{$part}{seq}}; for my $pair (@seq) { print " $pair->{op} $pair->{reg} ".length($pair->{data})."\n"; } } print "\n"; print "list:\n"; for my $item (@list) { print "$item\n"; } # ==== perform ==== my $flh; open $flh, '>', 'scripts/list.txt' || die "Can't write to 'scripts/list.txt': $!"; for my $i (0..$#list) { my ($name, $arch, $project_name, $project_major, $category, $platform_suffix, $scenario_suffix, $github_repo, $jenkins_platform); my $item = $list[$i]; my ($project, $repo, $platform, $component, $scenario) = $item=~/project=(.*) repo=(.*) platform=(.*) component=(.*) scenario=(.*)/; my ($os, $arch_bits); if ($platform =~ /(.*)-(.*)/) { ($os, $arch_bits) = ($1, $2); } else { $os = $platform; $arch_bits = undef; } if ($platform =~ /.*-32/) { $arch = 'i386'; } else { $arch = 'x86_64'; } if (not exists $projects{$project}) { die "unknown project '${project}' in '$item'"; } if (not exists $platforms{$platform}) { die "unknown platform '${platform}' in '$item'"; } if (not exists $scenarios{$scenario}) { die "unknown scenario '${scenario}' in '$item'"; } if (not exists $categories{$repo}) { die "unknown repo '${repo}' in '$item'"; } $project_name=$projects{$project}; ($project_major) = $project =~ /[a-z]*(.*)/; $platform_suffix = $platforms{$platform}; $scenario_suffix = $scenarios{$scenario}; $category = uc($project).'-'.$categories{$repo}; if ($component eq 'canl') { $github_repo = 'canl-c'; } elsif ($component eq 'gridsite') { $github_repo = 'gridsite'; } elsif ($component eq 'lb') { $github_repo = 'glite-lb'; } elsif ($component eq 'px') { $github_repo = 'proxyrenewal'; } if ($os eq 'sl6') { $jenkins_platform = "epel-6-$arch"; } elsif ($os eq 'sl7') { $jenkins_platform = "epel-7-$arch"; } elsif ($os eq 'fedora') { $jenkins_platform = "fedora-rawhide-$arch"; } elsif ($platform =~ /^deb(\d+)/) { $jenkins_platform = "debian-$1-$arch"; } elsif ($platform =~ /^ubnt(\d+)/) { $jenkins_platform = "ubuntu-$1-$arch"; } $name = "$project-$repo-$scenario_suffix$component-$platform_suffix.sh"; $data = ""; for my $part (sort keys %parts) { @seq = @{$parts{$part}{seq}}; $section = ""; for my $pair (@seq) { if ($item =~ $pair->{reg}) { if ($pair->{op} eq "+") { if ($section) { $section .= "\n"; } $section .= $pair->{data}; #print "$name: added $pair->{reg} into $parts{$part}{section}\n"; } elsif ($pair->{op} eq ".") { $section .= $pair->{data}; #print "$name: concated $pair->{reg} into $parts{$part}{section}\n"; } else { #print "$name: assigned $pair->{reg} into $parts{$part}{section}\n"; $section = $pair->{data}; } } } if ($data and $section) { $data .= "\n"; } $data .= $section; #print "\n"; } $data =~ s/\@ARCH\@/$arch/g; $data =~ s/\@ARCH_BITS\@/$arch_bits/g; $data =~ s/\@OS\@/$os/g; $data =~ s/\@PROJECT_NAME\@/$project_name/g; $data =~ s/\@PROJECT_MAJOR\@/$project_major/g; $data =~ s/\@COMPONENT\@/$component/g; $data =~ s/\@GITHUB_REPO\@/$github_repo/g; $data =~ s/\@JENKINS_PLATFORM\@/$jenkins_platform/g; $data =~ s,\@WGET\@,wget --no-verbose --ca-directory=/etc/grid-security/certificates,g; my @a = split /\n/, $data; my @b = grep { /^(wget|rpm) .*http.?\/\// } @a; my @c = map { /^(wget|rpm) .*(http.*\/\/[^ ]*).*/; $2 } @b; #for my $url (@c) { print "URL: $url\n"; } my @d = map { 'wget --no-verbose --no-check-certificate -O /tmp/test '.$_ } @c; my $e = join ' && ', @d; $e =~ s/\&\& $//; #print "pretest: $e\n"; open $fh, '>', "scripts/".$name || die "Can't write into '$name': $!"; printf $fh "%s", "#CATEGORY=$category\n"; printf $fh "%s", "#PRETEST=\"$e\"\n" if ($e); printf $fh "%s", $data; close $fh; if (not $platform =~ /.*-.*/ and $platform =~ /deb7/) { $platform = 'deb7-64'; } if (not $platform =~ /.*-.*/ and $platform =~ /deb8/) { $platform = 'deb8-64'; } if ($platform =~ /.*-.*/) { printf $flh "%s\t%s\n", $item, $name; } else { printf $flh "%s\t%s\n", "project=$project repo=$repo platform=$platform-32 component=$component scenario=$scenario", $name; printf $flh "%s\t%s\n", "project=$project repo=$repo platform=$platform-64 component=$component scenario=$scenario", $name; } } close $flh;