From: František Dvořák Date: Tue, 12 Sep 2006 18:20:18 +0000 (+0000) Subject: add attributes to view, special view for regtime attribute, X-Git-Tag: glite-jp-index_R_1_2_2~5 X-Git-Url: http://scientific.zcu.cz/git/?a=commitdiff_plain;h=4e2c2026d7b718ecb9efa951ecc3aef627c9cf9e;p=jra1mw.git add attributes to view, special view for regtime attribute, corrected not really tested query 4., implemented 5. query --- diff --git a/org.glite.jp.index/examples/pch06/pch.pm b/org.glite.jp.index/examples/pch06/pch.pm index 2e054ab..e4bb6db 100644 --- a/org.glite.jp.index/examples/pch06/pch.pm +++ b/org.glite.jp.index/examples/pch06/pch.pm @@ -17,6 +17,9 @@ our $jpsys='http://egee.cesnet.cz/en/Schema/JP/System'; our $jpwf='http://egee.cesnet.cz/en/Schema/JP/Workflow'; our $jplbtag='http://egee.cesnet.cz/en/WSDL/jp-lbtag'; +our @view_attributes=("$pch::jplbtag:IPAW_STAGE", "$pch::jplbtag:IPAW_PROGRAM", "$pch::jplbtag:IPAW_PARAM", "$pch::jplbtag:IPAW_INPUT", "$pch::jplbtag:IPAW_OUTPUT", "$pch::lbattr:CE", "$pch::lbattr:parent", "$pch::jpsys:regtime"); +#"$pch::lbattr:host", # TODO: to index server + our $debug = 0; our $err = 0; diff --git a/org.glite.jp.index/examples/pch06/query1.pl b/org.glite.jp.index/examples/pch06/query1.pl index 7bc938e..9b155b1 100644 --- a/org.glite.jp.index/examples/pch06/query1.pl +++ b/org.glite.jp.index/examples/pch06/query1.pl @@ -99,12 +99,17 @@ foreach my $jobid (sort { $according_jobs{$b} <=> $according_jobs{$a} } keys %ac print "jobid $jobid:\n"; # query & output all desired atributes - foreach my $attr ("$pch::jplbtag:IPAW_STAGE", "$pch::jplbtag:IPAW_PROGRAM", "$pch::jplbtag:IPAW_PARAM", "$pch::jplbtag:IPAW_INPUT", "$pch::jplbtag:IPAW_OUTPUT", "$pch::lbattr:CE") { + foreach my $attr (@pch::view_attributes) { my @attrs; my $attr_name = $attr; $attr_name =~ s/.*://; @attrs = pch::psquery($ps, $jobid, $attr); - print " attr $attr_name: "; print join(", ", @attrs); print "\n"; + print " attr $attr_name: "; + if ($attr eq "$pch::jpsys:regtime") { + print gmtime(@attrs[0])." (".join(", ", @attrs).")\n"; + } else { + print join(", ", @attrs)."\n"; + } } print "\n"; diff --git a/org.glite.jp.index/examples/pch06/query2.pl b/org.glite.jp.index/examples/pch06/query2.pl index 7143434..e189b59 100644 --- a/org.glite.jp.index/examples/pch06/query2.pl +++ b/org.glite.jp.index/examples/pch06/query2.pl @@ -73,7 +73,7 @@ foreach my $jobid (@according_jobs) { @program = pch::psquery($ps, $jobid, "$pch::jplbtag:IPAW_PROGRAM"); die "More program names of $jobid?" if ($#program > 0); if ($program[0] eq $program_name) { - print "$jobid is $program_name, stop here\n"; + print "$jobid is $program_name, stop here\n" if $debug; next; } @@ -108,12 +108,17 @@ foreach my $jobid (sort { $according_jobs{$b} <=> $according_jobs{$a} } keys %ac print "jobid $jobid:\n"; # query & output all desired atributes - foreach my $attr ("$pch::jplbtag:IPAW_STAGE", "$pch::jplbtag:IPAW_PROGRAM", "$pch::jplbtag:IPAW_PARAM", "$pch::jplbtag:IPAW_INPUT", "$pch::jplbtag:IPAW_OUTPUT", "$pch::lbattr:CE") { + foreach my $attr (@pch::view_attributes) { my @attrs; my $attr_name = $attr; $attr_name =~ s/.*://; @attrs = pch::psquery($ps, $jobid, $attr); - print " attr $attr_name: "; print join(", ", @attrs); print "\n"; + print " attr $attr_name: "; + if ($attr eq "$pch::jpsys:regtime") { + print gmtime(@attrs[0])." (".join(", ", @attrs).")\n"; + } else { + print join(", ", @attrs)."\n"; + } } print "\n"; diff --git a/org.glite.jp.index/examples/pch06/query3.pl b/org.glite.jp.index/examples/pch06/query3.pl index 2546707..ba4098f 100644 --- a/org.glite.jp.index/examples/pch06/query3.pl +++ b/org.glite.jp.index/examples/pch06/query3.pl @@ -15,8 +15,7 @@ use Data::Dumper; my $ps='https://skurut1.cesnet.cz:8901'; my $is='https://skurut1.cesnet.cz:8902'; -my @view_attributes = ("$pch::jplbtag:IPAW_STAGE", "$pch::jplbtag:IPAW_PROGRAM", "$pch::jplbtag:IPAW_PARAM", "$pch::jplbtag:IPAW_INPUT", "$pch::jplbtag:IPAW_OUTPUT", "$pch::lbattr:CE"); -my @attributes = ("$pch::jpsys:jobId", "$pch::jpwf:ancestor", @view_attributes); +my @attributes = ("$pch::jpsys:jobId", "$pch::jpwf:ancestor", @pch::view_attributes); my @according_jobs = (); # sequencially jobid list my %according_jobs = (); # hash jobid list @@ -100,14 +99,18 @@ foreach my $jobid (sort { $according_jobs{$b}{attributes}{"$pch::jplbtag:IPAW_ST print "jobid $jobid:\n"; # query & output all desired atributes - foreach my $attr (@view_attributes) { + foreach my $attr (@pch::view_attributes) { my $attr_name = $attr; $attr_name =~ s/.*://; print " attr $attr_name: "; if (exists $attributes{$attr}) { my %attr = %{$attributes{$attr}}; - print join(", ", @{$attr{value}}); print "\n"; + if ($attr eq "$pch::jpsys:regtime") { + print gmtime($attr{value}[0])." (".join(", ", @{$attr{value}}).")\n"; + } else { + print join(", ", @{$attr{value}})."\n"; + } } else { print "N/A\n"; } diff --git a/org.glite.jp.index/examples/pch06/query4.pl b/org.glite.jp.index/examples/pch06/query4.pl index 604ebd6..0d82890 100644 --- a/org.glite.jp.index/examples/pch06/query4.pl +++ b/org.glite.jp.index/examples/pch06/query4.pl @@ -8,7 +8,7 @@ # "-m 12" of align_warp) that ran on a Monday. # # call: -# ./query4.pl OUTPUT_FILE_NAME 2>/dev/null +# ./query4.pl 2>/dev/null # use strict; @@ -21,9 +21,8 @@ my $program_name='align_warp'; my $program_params='-m 12'; my $runday=1; my @view_attributes = ("$pch::jplbtag:IPAW_STAGE", "$pch::jplbtag:IPAW_PROGRAM", "$pch::jplbtag:IPAW_PARAM", "$pch::jplbtag:IPAW_INPUT", "$pch::jplbtag:IPAW_OUTPUT", "$pch::lbattr:CE"); -my @attributes = ("$pch::jpsys:jobId", "$pch::jpwf:ancestor", @view_attributes); +my @attributes = ("$pch::jpsys:jobId", "$pch::jpsys:regtime", @view_attributes); -my @according_jobs = (); # sequencially jobid list my %according_jobs = (); # hash jobid list my $according_count = 0; @@ -42,50 +41,37 @@ my @jobs = pch::isquery($is, [ print Dumper(@jobs) if ($debug); die "...so exit on error" if ($pch::err); + # -# initial set from index server +# check found all jobs # +$according_count = 0; foreach my $job (@jobs) { my %job = %$job; - my %attributes = %{$job{attributes}}; + my @time; - if (!exists $according_jobs{$job{jobid}}) { - push @according_jobs, $job{jobid}; - $according_jobs{$job{jobid}} = \%job; - } -} -undef @jobs; + print "Handling $job{jobid} ($according_count.)\n" if ($debug); - -# -# collect all jobs (tree browsing) -# -$according_count = 0; -foreach my $jobid (@according_jobs) { - my @ancs; - - print "Handling $jobid (position $according_count)\n" if ($debug); - @ancs = pch::isquery($is, [["$pch::jpwf:successor", ['EQUAL', "$jobid"]]], \@attributes); - die "...so exit on error" if ($pch::err); - - for my $anc (@ancs) { - my %anc = %$anc; - print "Considered: $anc{jobid}\n" if ($debug); - if (!exists $according_jobs{$anc{jobid}}) { - $according_jobs{$anc{jobid}} = \%anc; - push @according_jobs, $anc{jobid}; - print "Added $anc{jobid} to $#according_jobs\n" if ($debug); - } - else { - print "Already existing $anc{jobid}\n" if ($debug); + @time =@{ $job{attributes}{"$pch::jpsys:regtime"}{value}}; + my @timesep = gmtime($time[0]); + if ($timesep[6] == $runday) { + if (!exists $according_jobs{$job{jobid}}) { + $according_jobs{$job{jobid}} = \%job; + print "Added $job{jobid}\n" if $debug; + } else { + print "Already existing $job{jobid}\n" if $debug; } + } else { + print "Job $job{jobid} ran at day $timesep[6] (0=Sun, ...): ".gmtime($time[0])."\n" if $debug; } + $according_count++; } +undef @jobs; # -# queries on result set +# print the result set # print "Results\n"; print "=======\n"; @@ -94,28 +80,25 @@ foreach my $jobid (sort { $according_jobs{$b}{attributes}{"$pch::jplbtag:IPAW_ST my %job = %{$according_jobs{$jobid}}; my %attributes = %{$job{attributes}}; - my @time = pch::psquery($ps, $jobid, "$pch::jpsys:regtime"); - my @timesep = gmtime($time[0]); - if ($timesep[6] == $runday) { - print "jobid $jobid:\n"; + print "jobid $jobid:\n"; - # query & output all desired atributes - foreach my $attr (@view_attributes) { - my $attr_name = $attr; $attr_name =~ s/.*://; + # output all desired atributes + foreach my $attr (@view_attributes) { + my $attr_name = $attr; $attr_name =~ s/.*://; - print " attr $attr_name: "; - if (exists $attributes{$attr}) { - my %attr = %{$attributes{$attr}}; + print " attr $attr_name: "; + if (exists $attributes{$attr}) { + my %attr = %{$attributes{$attr}}; - print join(", ", @{$attr{value}}); print "\n"; + if ($attr eq "$pch::jpsys:regtime") { + print gmtime($attr{value}[0])." (".join(", ", @{$attr{value}}).")\n"; } else { - print "N/A\n"; + print join(", ", @{$attr{value}})."\n"; } + } else { + print "N/A\n"; } - print " attr REGTIME: ".gmtime($time[0])." (".join(", ", @time).")\n"; - - print "\n"; - } else { - print "Job $jobid ran at day $timesep[6] (0=Sun, ...): ".gmtime($time[0])."\n" if $debug; } + + print "\n"; } diff --git a/org.glite.jp.index/examples/pch06/query5.pl b/org.glite.jp.index/examples/pch06/query5.pl new file mode 100644 index 0000000..d621e64 --- /dev/null +++ b/org.glite.jp.index/examples/pch06/query5.pl @@ -0,0 +1,133 @@ +#! /usr/bin/perl + +# +# 5. query: +# +# Find all Atlas Graphic images outputted from workflows where at least one of +# the input Anatomy Headers had an entry global maximum=4095. The contents of +# a header file can be extracted as text using the scanheader AIR utility. +# +# call: +# ./query5.pl 2>/dev/null +# + +use strict; +use pch; +use Data::Dumper; + +my $ps='https://skurut1.cesnet.cz:8901'; +my $is='https://skurut1.cesnet.cz:8902'; +my $program_name='align_warp'; +my $end_program_name='convert'; +my $atlas_image_program_name='convert'; +my $header="GLOBAL_MAXIMUM=4095"; # test for exact equal (scripts already prepared it) + +my @according_jobs = (); # sequencially jobid list +my %according_jobs = (); # hash jobid list +my $according_count = 0; + + +# debug calls +$pch::debug = 1; +my $debug = 1; + +# +# find out processes with given name and parameters +# +my @jobs = pch::isquery($is, [ + ["$pch::jplbtag:IPAW_PROGRAM", ['EQUAL', "$program_name"]], + ["$pch::jplbtag:IPAW_HEADER", ['EQUAL', "$header"]], +], \@pch::view_attributes); +print Dumper(@jobs) if ($debug); +die "...so exit on error" if ($pch::err); + +# +# initial set of DAGs from index server +# +foreach my $job (@jobs) { + my %job = %$job; + my %attributes = %{$job{attributes}}; + my $dagjobid = $attributes{"$pch::lbattr:parent"}{value}[0]; + + if (!exists $according_jobs{$dagjobid}) { + %job = (); + push @according_jobs, $dagjobid; + # query to primary storage when searching by jobid + $job{jobid} = $dagjobid; + foreach my $attr (@pch::view_attributes) { + my @value; + + @value = pch::psquery($ps, $dagjobid, $attr); + if (defined @value) { $job{attributes}{$attr} = \@value; } + } + $according_jobs{$dagjobid} = \%job; + } +} +undef @jobs; + + +# +# collect all jobs (tree browsing down) +# +$according_count = 0; +foreach my $jobid (@according_jobs) { + my @succs; + + print "Handling $jobid (position $according_count)\n" if ($debug); + + if ($according_jobs{$jobid}{attributes}{"$pch::IPAW_PROGRAM"}[0] eq $end_program_name) { + print "It's $end_program_name\n" if $debug; + next; + } + + @succs = pch::isquery($is, [["$pch::jpwf:ancestor", ['EQUAL', "$jobid"]]], \@pch::view_attributes); + die "...so exit on error" if ($pch::err); + + for my $succ (@succs) { + my %succ = %$succ; + print "Considered: $succ{jobid}\n" if ($debug); + if (!exists $according_jobs{$succ{jobid}}) { + $according_jobs{$succ{jobid}} = \%succ; + push @according_jobs, $succ{jobid}; + print "Added $succ{jobid} to $#according_jobs\n" if ($debug); + } + else { + print "Already existing $succ{jobid}\n" if ($debug); + } + } + $according_count++; +} + + +# +# print the result set +# +print "Results\n"; +print "=======\n"; +print "\n"; +foreach my $jobid (sort { $according_jobs{$b}{attributes}{"$pch::jplbtag:IPAW_STAGE"}{value}[0] <=> $according_jobs{$a}{attributes}{"$pch::jplbtag:IPAW_STAGE"}{value}[0] } keys %according_jobs) { + my %job = %{$according_jobs{$jobid}}; + my %attributes = %{$job{attributes}}; + + print "jobid $jobid:\n"; + + # output all desired atributes + foreach my $attr (@pch::view_attributes) { + my $attr_name = $attr; $attr_name =~ s/.*://; + + print " attr $attr_name: "; + if (exists $attributes{$attr}) { + my %attr = %{$attributes{$attr}}; + + if ($attr eq "$pch::jpsys:regtime") { + print gmtime($attr{value}[0])." (".join(", ", @{$attr{value}}).")\n"; + } else { + print join(", ", @{$attr{value}})."\n"; + } + } else { + print "N/A\n"; + } + } + + print "\n"; +}