From 1d469d68d41d12fcf438a70458ac903613203782 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Jan=20Posp=C3=AD=C5=A1il?= Date: Sat, 15 Dec 2007 23:42:41 +0000 Subject: [PATCH] initial import --- org.glite.jp.doc/Makefile | 49 ++ org.glite.jp.doc/src/JPAG-Configuration.tex | 5 + org.glite.jp.doc/src/JPAG-Installation.tex | 7 + org.glite.jp.doc/src/JPAG-Introduction.tex | 5 + org.glite.jp.doc/src/JPAG-Running.tex | 5 + org.glite.jp.doc/src/JPAG-Troubleshooting.tex | 6 + org.glite.jp.doc/src/JPAG.tex | 45 ++ org.glite.jp.doc/src/JPDG-Introduction.tex | 6 + org.glite.jp.doc/src/JPDG.tex | 36 ++ org.glite.jp.doc/src/JPUG-Introduction.tex | 693 +++++++++++++++++++++++++ org.glite.jp.doc/src/JPUG-Tools.tex | 8 + org.glite.jp.doc/src/JPUG-UseCases.tex | 3 + org.glite.jp.doc/src/JPUG.tex | 43 ++ org.glite.jp.doc/src/copyright.tex | 24 + org.glite.jp.doc/src/definitions.tex | 17 + org.glite.jp.doc/src/egee.cls | 507 ++++++++++++++++++ org.glite.jp.doc/src/frontmatter.tex | 40 ++ org.glite.jp.doc/src/gui.tex | 3 + org.glite.jp.doc/src/jpimporter.tex | 2 + org.glite.jp.doc/src/lbjp.bib | 709 ++++++++++++++++++++++++++ 20 files changed, 2213 insertions(+) create mode 100644 org.glite.jp.doc/Makefile create mode 100644 org.glite.jp.doc/src/JPAG-Configuration.tex create mode 100644 org.glite.jp.doc/src/JPAG-Installation.tex create mode 100644 org.glite.jp.doc/src/JPAG-Introduction.tex create mode 100644 org.glite.jp.doc/src/JPAG-Running.tex create mode 100644 org.glite.jp.doc/src/JPAG-Troubleshooting.tex create mode 100644 org.glite.jp.doc/src/JPAG.tex create mode 100644 org.glite.jp.doc/src/JPDG-Introduction.tex create mode 100644 org.glite.jp.doc/src/JPDG.tex create mode 100644 org.glite.jp.doc/src/JPUG-Introduction.tex create mode 100644 org.glite.jp.doc/src/JPUG-Tools.tex create mode 100644 org.glite.jp.doc/src/JPUG-UseCases.tex create mode 100644 org.glite.jp.doc/src/JPUG.tex create mode 100644 org.glite.jp.doc/src/copyright.tex create mode 100644 org.glite.jp.doc/src/definitions.tex create mode 100644 org.glite.jp.doc/src/egee.cls create mode 100644 org.glite.jp.doc/src/frontmatter.tex create mode 100644 org.glite.jp.doc/src/gui.tex create mode 100644 org.glite.jp.doc/src/jpimporter.tex create mode 100644 org.glite.jp.doc/src/lbjp.bib diff --git a/org.glite.jp.doc/Makefile b/org.glite.jp.doc/Makefile new file mode 100644 index 0000000..6a693d3 --- /dev/null +++ b/org.glite.jp.doc/Makefile @@ -0,0 +1,49 @@ +# Default values +top_srcdir=. +stagedir=. +globalprefix=glite +lbprefix=lb +package=glite-jp-doc +version=0.0.0 +PREFIX=/opt/glite + +-include Makefile.inc + +VPATH = ${top_srcdir}/src +KPATH = TEXINPUTS=".:$(VPATH)//:" +KPATHBIB = BIBINPUTS=".:$(VPATH)//:" + +LATEX = $(KPATH) latex +PDFLATEX = $(KPATH) pdflatex +BIBTEX = $(KPATHBIB) bibtex +DVIPS = $(KPATH) dvips +AT3=${stagedir}/sbin/glite-lb-at3 + +default all: generate JPUG.pdf JPAG.pdf JPDG.pdf + +generate: + +# %.dvi: %.tex +# $(LATEX) $< +# $(BIBTEX) `basename $< .tex` +# $(LATEX) $< +# $(LATEX) $< +# +# %.ps: %.dvi +# $(DVIPS) -ta4 -o $@ $< + +%.pdf: %.tex + $(PDFLATEX) $< + $(BIBTEX) `basename $< .tex` + $(PDFLATEX) $< + $(PDFLATEX) $< + +%.tex: %.tex.T + rm -f $@ + ${AT3} $< >$@ || rm -f $@ + chmod -w $@ >/dev/null + +clean: + rm -rvf JPUG* JPAG* JPDG* + +.PHONY: all clean purge ps pdf diff --git a/org.glite.jp.doc/src/JPAG-Configuration.tex b/org.glite.jp.doc/src/JPAG-Configuration.tex new file mode 100644 index 0000000..e08ce37 --- /dev/null +++ b/org.glite.jp.doc/src/JPAG-Configuration.tex @@ -0,0 +1,5 @@ +\section{Configuration} +\TODO{} + +\subsection{Setting up the MySQL database} + diff --git a/org.glite.jp.doc/src/JPAG-Installation.tex b/org.glite.jp.doc/src/JPAG-Installation.tex new file mode 100644 index 0000000..fb1b170 --- /dev/null +++ b/org.glite.jp.doc/src/JPAG-Installation.tex @@ -0,0 +1,7 @@ +\section{Installation} +\TODO{} + +\subsection{Complete RPMs description} +\subsection{Daemons description} +\subsection{CLI tools description} + diff --git a/org.glite.jp.doc/src/JPAG-Introduction.tex b/org.glite.jp.doc/src/JPAG-Introduction.tex new file mode 100644 index 0000000..7843699 --- /dev/null +++ b/org.glite.jp.doc/src/JPAG-Introduction.tex @@ -0,0 +1,5 @@ +\section{Introduction} +\TODO{This document should contain:} + +\subsection{Deployment scenarios} + diff --git a/org.glite.jp.doc/src/JPAG-Running.tex b/org.glite.jp.doc/src/JPAG-Running.tex new file mode 100644 index 0000000..fdb6ae8 --- /dev/null +++ b/org.glite.jp.doc/src/JPAG-Running.tex @@ -0,0 +1,5 @@ +\section{Running and stopping the services} +\TODO{} + +\subsection{Tests if everything works properly} + diff --git a/org.glite.jp.doc/src/JPAG-Troubleshooting.tex b/org.glite.jp.doc/src/JPAG-Troubleshooting.tex new file mode 100644 index 0000000..c4397a4 --- /dev/null +++ b/org.glite.jp.doc/src/JPAG-Troubleshooting.tex @@ -0,0 +1,6 @@ +\section{Troubleshooting} +\TODO{} + +\subsection{Debugging} +\subsection{Fine tuning the performance} + diff --git a/org.glite.jp.doc/src/JPAG.tex b/org.glite.jp.doc/src/JPAG.tex new file mode 100644 index 0000000..7652f84 --- /dev/null +++ b/org.glite.jp.doc/src/JPAG.tex @@ -0,0 +1,45 @@ +\documentclass{egee} +%\usepackage{doxygen} + +\input{definitions} + +\title{Job Provenance} +\Subtitle{Administrator's Guide} +\author{CESNET EGEE II JRA1 team} +\DocIdentifier{EGEE-II....} +\Date{\today} +\Activity{JRA1: Middleware Engineering and Integration} +\DocStatus{DRAFT} +\Dissemination{PUBLIC} +\DocumentLink{http://...} + +\Abstract{ This administrator's guide explains how to administer the Job +Provenance (\JP) service. Several deployment scenarios are described together +with the installation, configuration, running and troubleshooting steps. } + +\begin{document} + +\input{frontmatter} +\newpage +\tableofcontents + +\newpage +\input{JPAG-Introduction} + +\newpage +\input{JPAG-Installation} + +\newpage +\input{JPAG-Configuration} + +\newpage +\input{JPAG-Running} + +\newpage +\input{JPAG-Troubleshooting} + +\nocite{jgc} +\bibliographystyle{unsrt} +\bibliography{lbjp} + +\end{document} diff --git a/org.glite.jp.doc/src/JPDG-Introduction.tex b/org.glite.jp.doc/src/JPDG-Introduction.tex new file mode 100644 index 0000000..af8b84c --- /dev/null +++ b/org.glite.jp.doc/src/JPDG-Introduction.tex @@ -0,0 +1,6 @@ +\section{Introduction} + +\TODO{Best practices :)} + + +\section{Web Services Interface} diff --git a/org.glite.jp.doc/src/JPDG.tex b/org.glite.jp.doc/src/JPDG.tex new file mode 100644 index 0000000..dd877a5 --- /dev/null +++ b/org.glite.jp.doc/src/JPDG.tex @@ -0,0 +1,36 @@ +\documentclass{egee} + +\input{definitions} + +\title{Job Provenance} +\Subtitle{Developer's Guide} +\author{CESNET EGEE II JRA1 team} +\DocIdentifier{EGEE-II....} +\Date{\today} +\Activity{JRA1: Middleware Engineering and Integration} +\DocStatus{DRAFT} +\Dissemination{PUBLIC} +\DocumentLink{http://...} + +\Abstract{This developer's guide explains how to use the Job Provenance (\JP) +service API, namely the Web Services Interface is described in details together with +programing examples. } + +\begin{document} + +\input{frontmatter} +\tableofcontents + +\newpage +\input{JPDG-Introduction} + +%\newpage +%\input{web_services} + +\newpage +\nocite{jgc} +\bibliographystyle{unsrt} +\bibliography{lbjp} + +\end{document} + diff --git a/org.glite.jp.doc/src/JPUG-Introduction.tex b/org.glite.jp.doc/src/JPUG-Introduction.tex new file mode 100644 index 0000000..36b8e6b --- /dev/null +++ b/org.glite.jp.doc/src/JPUG-Introduction.tex @@ -0,0 +1,693 @@ +\section{Introduction} + +The Job Provenance (\JP) service is primarily designed to provide a permanent +storage and advanced querying interface to the data about Grid jobs and +the environment they were run in. This information is to be used for +statistical purposes, lookup for patterns in the Grid behavior and also +for job re-submission. + +The Job Provenance extends the data model specified by the \LB service +with additional information about each job---most specifically the input +and output data files---and also information about the run time +environment. + +The Job Provenance must fulfill rather contradictory requirements. It +must keep detailed information about each job, the environment the job +run in and the affected files, as possible. On the other hand, being a +permanent service, the job records must be kept reasonably small +to fit into reasonable sized storage system. Given the expected number of +jobs on large scale Grids---\eg the EGEE already +reports\footnote{\url{http://egee-jra2.web.cern.ch/EGEE-JRA2/QoS/JobsMetrics/JobMetrics.htm}} +20k jobs per day, that is 7.5M jobs per year, a number of jobs before the +large experiments will be deployed---the \JP must also support very +efficient searching and querying features. Another problem is associated +with the long life span of the \JP service. It must be expected that the +data formats will change over the time, while the \JP is expected to deal +with old and new data formats in a uniform way. They can be achieved via +extensibility of the JP data model. + +As the data collection serviced by the JP will extensively grow, it is +impossible to rely only on the primary data when navigating through it. +Users must be able to add annotations to individual job records and these +annotations serve two primary purposes---to help in organizing the \JP +data and to be a source of additional information, not provided directly +by the automated collection of primary data. Even annotations must follow +the WORM (write once read many times) semantics, as they are always added +on top of the already stored data, never re-writing the old annotations. +Work with the most recent set of annotations as well as ability to +inspect the history of annotations must be supported. + +Fig.~\ref{fig:psinter} depicts interaction between Job Provenance and +other Grid middleware components (on the example of the gLite +infrastructure). +\begin{figure}[ht] + \centering + \includegraphics[scale=0.5]{images/JP-interactions} + \caption{Data flow into gLite Job Provenance} + \label{fig:psinter} +\end{figure} + +\subsection{Concepts} +\ludek{\TODO{Obrazky moc nezapadaji do soucasne struktury textu :-(}} + +\subsubsection{Data gathering}% +\label{data} +%\begin{comment} +%\todo{job record, job attributes} \todo{open design, scalable} +%\todo{we have very small persistent record (jobid, owner, timescale), +% set of associated files and we maintain corresponding plugins (by what?) to +% interpret them; metadata are mined, indexed and provided on demand} +%\todo{attribute namespaces---a glue to source of metadata} +%\end{comment} + +The primary data organization in \JP is on a per job basis, a concept +taken from the \LB data organization. Every data item stored in JP is +associated to a~particular Grid job. As the overall storage capacity +requirements may become enormous, we store only volatile data which are +neither stored reliably elsewhere nor are reproducible by the job. +The data gathered from the gLite middleware fall into the following categories: +\begin{itemize} +\item job inputs, directly required for job re-running +\begin{itemize} +\item complete job description (JDL) as submitted to WMS +\item miscellaneous input files (gLite WMS input sandbox) provided by the user +(but job input files from remote storage \emph{are not} copied to JP) +\end{itemize} +\item job execution track, witnessing the environment of job execution +\begin{itemize} +\item complete \LB data, for example when and where the job was + planned and executed, how many times and for what reasons it was + resubmitted etc. +\item ``measurements'' on computing elements, \eg versions of installed +software, environment settings etc. +\end{itemize} +\end{itemize} +In addition, the service allows the user to add arbitrary annotations to +a~job in a form of ``name = value'' pairs. +These annotations can be recorded either during the job execution or at any time +afterward. +Besides providing information on the job (\eg it was a~production-phase +job of a particular experiment) these annotations may carry +information on relationships between the job and other entities +like external datasets, forming the desired data provenance record. + +%\TODO{zduraznit WORM semantiku + citace na IPAW} +Once a~piece of data is recorded for a~job, it can be never updated or +replaced. +New values can be recorded% +\footnote{It seems to make sense only for the annotations, not the +middleware data, and the current implementation makes this restriction. +However, it can be relaxed without principal impact.} +but the old values are always preserved. +Consequently the recorded history cannot be lost. + + +\subsubsection{Data representation}% +\label{attrib} + +The \JP concept distinguishes between two views on the processed data. +The data are stored in the \JP in the \emph{raw representation}. +Two input modes are assumed, depending mainly on the size +and structure of the data: +\begin{itemize} +\item Small size \emph{tags} in a form of ``name = value'' pairs, +enter the system via its primary interface (as a~web service operation +in the current implementation). +``Value'' is assumed to be a literal, without any structure that \JP should +be aware of. + +\item \emph{Bulk files}, \eg the complete dump of \LB data or the job input +sandbox, are uploaded via a~suitable transfer protocol% +\footnote{The current implementation supports \texttt{gsiftp://} only but +other protocols can be easily added.}. +Files are supposed to be structured. +However, they are stored ``as is'', and upon upload they are annotated with +format identification, including version of the format. +JP allows installing plugins that handle particular file formats (see bellow), +understanding the file structure and extracting required information. +\end{itemize} + +Most data manipulation is done using the \emph{logical view}. +Any piece of information stored in JP is represented +as a~value of a particular named \emph{attribute}. Tags (user +annotations) map to attributes in a~straightforward way, name and value +of the tags becoming name and value of an attribute. An uploaded file is +usually a~source of multiple attributes, which are automatically extracted +via \emph{plugins}. JP defines a~\emph{file-type plugin interface API}. The +task of the plugin is parsing a~particular file type and providing calls +to retrieve attribute values. + +To avoid naming conflicts even with future attributes, +an attribute name always falls into a~namespace. +Currently we declare three different namespaces: for JP system attributes +(\eg job owner or +registration time), attributes inherited from \LB, and unqualified user tags. + +\ludek{ +This representation unifies the user annotations, the ``system'' +middleware data and information extracted from uploaded files into +a~single view.} + +\iffalse +An attribute value must be always representable in the form of a~printable +string. +If it is not the case for a~native attribute type, +converting rules must be provided by a~plugin (dynamically loadable library) +implementing a~specified API. +\fi + +We keep the scheme symmetric, which means that none of the currently declared attribute +namespaces is privileged in any sense. +However, it may present a~vulnerability\Dash a~malicious user may try +to override a~\JP system attribute using the user annotation interface. +Therefore each attribute value carries a~further \emph{origin classification}: +currently \emph{system}, \emph{user} (recorded as tag), and +\emph{file} (found in an uploaded file). + +Finally, as JP does not support updating data intentionally +(see Sect~\ref{data}), multiple values of an attribute are allowed. +The order in which the values were recorded can be reconstructed from +timestamps attached to each value, getting the ``attribute update'' semantics +if it is required. + +Attributes, representing the logical view, is the only way to specify queries on JP. +However, once the user knows an~actual jobid, bulk files can be retrieved +in the raw form, too (assumed to be useful in the case of input sandboxes +reused for job re-execution). + +\subsubsection{Layered architecture}% +\label{layered} +\JP is formed of two classes of services: the permanent \emph{Primary Storage} +(JPPS) accepts and stores job data +while possibly volatile and configurable \emph{Index Servers} (JPIS) +provide an optimized querying and data-mining interface to the end-users. + +The expected large amount of stored data yields a requirement on +maximal compactness of the storage. +The raw data should be compressed, +and the set of metadata kept with each job must be minimal. +\JP defines a~set of \emph{primary attributes} which are maintained +by JPPS for each job. +Jobid is the only mandatory primary attribute, +other suggested ones are the job owner, submission time, +and the virtual organization. +All other attributes are retrieved from the raw data only when requested. + +The restricted set of primary attributes prohibits user queries to be served +by the JPPS directly (with the only exception of the known jobid). +Due to the expected low selectivity of primary attributes such queries +would result in processing large number of job records, overloading +the server when the queries became frequent. + +These contradictory requirements (compactness vs. performance) had to be +resolved at another component layer. +The main idea is preprocessing the huge \JP dataset with several queries, +each of them covering a~superset of one expected class of user queries +(\eg jobs of particular VO, submitted in certain period). +If these super-queries are chosen carefully, they retrieve only a~small +fraction of the primary data. Their results can thus be stored (or cached) +in a~richer form, including various indices, hence being suitable +for fast response to user queries. +Querying JPPS in this way and maintaining the cache of the query result +is the task of Index Servers in the \JP architecture. + +Relationship of JPPS and JPIS is a many-to-many---a~single JPIS can query +multiple JPPS's and vice versa, a~single JPPS is ready to feed multiple JPIS's. + +The query conditions restrict the dataset in terms of the matching job records. +Similarly, the query specifies a~set of attributes to retrieve, +reducing also considerably the amount of retrieved data per each matching job. + +Index Servers query JPPS in two modes: in \emph{batch mode} JPIS is populated +with all JPPS content matching the query. In this way JPIS can be created +from scratch, despite the ope-ration is rather heavy for JPPS. +On the other hand, the \emph{incremental mode} allows JPIS to subscribe with +JPPS to receive new matching records as well as updates to already stored +records whenever new data arrive to JPPS. +This mode allows existing JPIS to be kept up to date while it is +still lightweight for JPPS. + +Finally, the described layer of JPIS's needn't be the only one. +The architecture (despite not the current implementation) allows +building another layer of JPIS's with many-to-many relationship with +the previous layer instances, combining their data, providing support +for other specific user queries etc. + +\subsection{Prototype implementation} + +The \JP prototype implementation% +\footnote{a part of the gLite middleware 3.x} +follows the described architecture. + +\subsubsection{Primary Storage}% +\label{primary} +\iffalse +The JP \emph{Primary Storage} (JPPS) is a~permanent service responsible +for gathering the job data and their long-term archival. +The primary data are kept in as compact form as possible, and +only minimal metadata (jobid and owner, registration time) are maintained +and indexed. +\fi + +A~single instance of JPPS, shown in Fig.~\ref{fig:psinter}, +is formed by a~front-end, exposing +its operations via a~web-service interface% +\footnote{Described in detail in ``EGEE Middleware +Design'',\\ \url{https://edms.cern.ch/document/487871/}, +documented web service definitions can be found at \\ +\url{http://egee.cesnet.cz/en/WSDL/}}% +, and a~back-end, responsible +for actual data storage and providing the bulk file transfer interface. +The front-end metadata (the primary attributes for each job, +authorization information, and JPIS subscription data) +are stored in a~relational database (currently MySQL). + +The back-end uses Globus \texttt{gridftp} server enriched with authorization +callbacks accessing the same database to check whether a~user +is allowed to upload or retrieve a~given file. +Both the front- and back-ends share a~filesystem so that the file-type plugins +linked into the front-end access their files via POSIX~I/O. + +JPPS operations fall into the following categories: +\begin{itemize} +\item\emph{Job registration.} +Each job has to be explicitly registered with JP. +Currently the registration is done transparently by the \LB server +upon job submission (in parallel with the job registration in \LB, +though not blocking the job submission). + +\item\emph{Tag recording.} +Add user tags (annotations) in the ``name = value'' form to JP job records. + +\item\emph{Bulk file upload.} +File properties (type, optional name etc.) +are specified via the front-end interface, the upload itself goes +directly to the back-end (using \texttt{gridftp} transfer). + +\item\emph{Index Server feed} allows JPIS to ask for batch feed +as well as register for incremental updates. + +\item\emph{Data retrieval.} +The only direct data retrieval supported by JPPS is keyed by +the jobid. +Both individual attributes and the whole files can be retrieved. +\end{itemize} + +Primary Storage covers the first set of requirements specified +for a Job Provenance (see Sect.~\ref{jp:req}), \ie storing a~compact job +record, allowing the user to add annotations, and providing elementary +access to the data. + +\subsubsection{Index Server} +Index Servers are created, configured, and populated semi-dynamically +according to particular user community needs. +The configuration is formed by: +\begin{itemize} +\item one or more Primary Storages to contact, +\item conditions (expressed in terms of \JP attributes) +on jobs that should be retrieved, +\item list of attributes to be retrieved, +\item list of attributes to be indexed\Dash a~user query must refer +to at least one of these for performance reasons. +\end{itemize} +The set of attributes and the conditions specify the set of data that +is retrieved from JPPS, and they reflect the assumed pattern +of user queries. + +The current JPIS implementation keeps the data also in a~MySQL database. +Its schema is flexible, reflecting the server configuration +(columns are created to hold particular attribute value, as well as indices). + +The JPIS interface operations fall into the following categories: +\begin{itemize} +\item\emph{Responses to user queries.} +A~query specifies a~set of attributes to retrieve and +conditions determining the set of matching jobs. +The structure of the query language is identical to~\LB---% +it allows two-level nesting of comparisons of an attribute +with a~constant value. + +\item\emph{JPIS-JPPS communication} implements the data flow from JPPS to JPIS. + +\item\emph{Administrative} calls to change JPIS configuration +without interfering with its normal operation. +\end{itemize} + + +\iffalse % puvodni text, pouzity jen jako zdroj materialu v predchozim + +The role of \emph{Index Servers} (JPIS) is processing and re-arranging the data +from Primary Storage(s) into a~form suitable for frequent and complex user +queries. + +We can divide implemented operation into three categories: +\begin{description} +\item[Responses to user queries.] +The main purpose of JPIS is to enable user complex queries. + +\begin{itemize} +\item The \texttt{QueryJobs} operation enables to ask the user query and +specify which attributes of matched jobs are returned. +\end{itemize} + +\item[JPIS-JPPS communication.] Internal call enabling data exchange between. +JPIS and JPPS. +\begin{itemize} +\item The \texttt{UpdateJobs} Called by JPPS as a response to \texttt{FeedIndex} +request. Updates information on jobs in JPIS, according to what JPPS +currently knows. +\end{itemize} + +\item[Administration.] Admin calls for changing configuration on the fly. +\begin{itemize} +\item The \texttt{AddFeed} Called by JPIS admin tool to ask new primary storage server to feed it. Updates information on JPPS in index server, according to what JPPS currently knows. +\item The \texttt{GetFeedIDs} Called by JPIS admin tool to find out its open feeds. +\item The \texttt{DeleteFeed} Called by JPIS admin tool to remove one feed session. +\end{itemize} + +\end{description} + + +\subsubsection{Index Server} +\label{index} +%\begin{wrapfigure}{r}{.5\hsize} +\begin{figure} + \centering + \includegraphics[scale=0.5]{images/JP-query} + \caption{Index Server interactions} + \label{fig:query} +\end{figure} +%\end{wrapfigure} + + + +A~typical interaction is shown in Fig.~\ref{fig:query}.%\\[-8mm] + +\begin{enumerate} +\item The user queries one or more JPIS, receiving a~list of ID's +of matching jobs. +\item JPPS is directly queried for additional job attributes or URL's of +stored files. +\item The required files are retrieved. +\end{enumerate} + +The current format of the user query is a~list of lists of conditions. +A~condition is comparison (less, greater, equal) of an attribute +\wrt\ a~constant. Items of an~inner list must refer to the same attribute +and they are logically or-ed. +Finally the inner lists are logically and-ed. +According to our experience with the \LB\ service, +this query language is powerful enough to satisfy user needs +while simple enough to allow efficient implementation. + +Index Servers are created, configured, and populated semi-dynamically +according to particular user community needs. +The configuration is formed by: +\begin{itemize} +\item one or more Primary Storages to contact, +\item conditions on jobs that should be retrieved, +\item list of attributes to be retrieved, +\item list of attributes to be indexed\Dash a~user query must refer +to at least one of these for performance reasons. +\end{itemize} +The set of attributes and the conditions specify the set of data that +is retrieved from JPPS, and it reflects the assumed pattern +of user queries. +The amount of data fed into a~single JPIS instance is assumed to be +only a~fraction of data in JPPS, +both regarding the number of jobs, and the number of distinct attributes. + +Communication between JPIS and JPPS involves two +complementary web-service operations: +JPIS calls the \texttt{FeedIndex} operation of JPPS, +specifying the list of attributes and conditions. +Unlike the user queries, the query on JPPS is a~single and-ed list, +allowing less complex processing on JPPS where significantly larger +data set are involved. +JPPS responds by calling the \texttt{UpdateJobs} operation of JPIS +(repeatedly to split up large dataset). + +The following flags in the \texttt{FeedIndex} call specify the query mode: +\begin{itemize} +\item \emph{history}\Dash JPPS should process all its stored data, +giving the user the guaranty that if +her query is a~logical restriction of the JPIS configuration, +it returns a~complete result. +This type of query is usually necessary to populate JPIS but it imposes +rather high load on JPPS. +\item \emph{continuous}\Dash JPIS registers with JPPS for receiving +\emph{future updates} when data matching the query arrive. +This type of query allows JPIS to be kept up to date while imposing minimal +load on JPPS. +\end{itemize} + +The current JPIS implementation keeps the data also in a~MySQL database. +Its schema is flexible, reflecting the Server configuration +(columns are created to hold particular attribute value, as well as indices). +There is no prescribed relationship between Primary Storage and Index Server +installations. +An Index Server may retrieve data from multiple Primary Storages +and vice versa. + +\fi % konec materialu + +\subsubsection{Scalability and deployment} +Having evaluated a~random sample of \LB data on approx. 1000 jobs, +we claim that the usual size of a~complete \LB data dump varies +from 2\,kB to 100\,kB, with very rare exceptions (less than 1\,\%) +of sizes up to 5\,MB. +However, these plain text files contain repeating patterns and they can +be compressed with the ratio of 1:4--1:20 in the typical cases +and even higher for the large files. +Therefore the assumption of 10\,kB compressed \LB dump per job is a~fairly +safe upper limit. +Unfortunately, we were not able to do a~similar assessment for job sandbox +sizes. Expecting the sandboxes to contain only miscellaneous input files +we work with the hypothesis of 100\,kB--1\,MB sandbox size. + +The current statistics for the entire infrastructure of the EGEE project +report the rate of 20,000 jobs per day, while the middleware +performance challenges aim at one million jobs per day. + +\begin{table} +\begin{tabular}{r|c|c|c} +\textbf{job rate $\backslash$ size} & \textbf{10\,kB \LB} & \textbf{100\,kB sandbox} & \textbf{1\,MB sandbox} \\ +\hline +current 20\,k/day & 73\,GB/year & 730\,GB/year & 7.3\,TB/year \\ +challenge 1\,M/day & 3.6\,TB/year & (36\,TB/year) & (360\,TB/year) +\end{tabular} +\caption{Expected aggregate storage size (whole EGEE)} +\label{t:jpsize} +\end{table} + +\begin{table} +\begin{tabular}{r|c|c|c} +\textbf{job rate $\backslash$ size} & \textbf{10\,kB \LB} & \textbf{100\,kB sandbox} & \textbf{1\,MB sandbox} \\ +\hline +current 20\,k/day & 2.3\,kB/s & 23\,kB/s & 230\,kB/s \\ +challenge 1\,M/day & 115\,kB/s & (1.15\,MB/s) & (11.5\,MB/s) +\end{tabular} +\caption{Expected aggregate incoming data rate (whole EGEE)} +\label{t:jprate} +\end{table} + + +Tables~\ref{t:jpsize} and \ref{t:jprate} use the discussed numbers to derive +the per-year storage size and per-second incoming data rate requirements on Job +Provenance. +The sandbox numbers for the 1\,M job challenge are shown in parentheses +because of being rather hypothetical\Dash +in order to achieve the required job throughput at WMS side, +the jobs must be submitted in fairly large \emph{collections} +(chunks of approx. 100--10,000 individual jobs) that share a~single input +sandbox% +\footnote{This statement is based on informal discussions with WMS developers. +The targeted WMS instance throughput at the time of this manuscript preparation +was 1 sandbox-less job per second, \ie\ 84.6\,k such jobs per day (achieving +the 1\,M job rate with WMS clustering only), +while the sandbox handling overhead is considered +to be unsustainable at this rate.}. +Therefore the real aggregate storage and throughput requirements on \JP can be +reduced by the factor of at least 100. + +Despite these figures are aggregate for the whole huge EGEE infrastructure, +they clearly show that the requirements could be met even with a~single +reasonably sized server. + +JP is designed to support many-to-many relationship of JPPS and JPIS +instances. Therefore there are no strict design requirements on the +number and structure of installations. +However, for practical reasons (some emerge from Sect.~\ref{jpusage}), +it is desirable to keep just small number of well known +JPPS's permanent services. +Typical setup can be one \JP per a~larger virtual organization, or even +one \JP shared by several smaller ones. +The outlined numbers show that this approach should not face technical +limits. +On the other hand, JPIS's are expected to be set up and configured +semi-dynamically, according to the varying needs of even small user +communities. + + +\iffalse +Only limited number of JPPS installations must be deployed even on +a~large Grid to concentrate the provenance data. At most one JPPS per +a~virtual organization is envisaged for the EGEE environment. +This mean each JPPS must be able to deal with data on millions of jobs. The +typical size of an \LB\ dump is around 10\,kB per compressed record, +and gLite users are encouraged not to use large job sandboxes, too. +Consequently, the back-end storage requirements are at the order of 10-100\,GB. +JPPS metadata are formed by a~single tuple for +each job and for each file, with unique indices on jobid and file name. +The used MySQL database engine is capable to handle millions of such records. +\fi + + +\subsection{Use patterns}% +\label{jpusage} + +\subsubsection{Storing data} +Propagation of data from other middleware components to \JP is done +transparently. +The user may affect it indirectly by +specifying the destination JPPS and gathered data extent +(\eg whether to store the job to \JP at all, or which sandbox files to keep) +via parameters in the job description. These settings +may be overridden by the WMS or CE policies. + +The user stores data to \JP directly when recording annotations +(Sects.~\ref{data} and~\ref{primary}). \ludek{JPPS instance which stores +the information on the job must be known, see bellow.} + +\subsubsection{Single job processing} +When full information on a~particular job is required (\eg for the +job re-execution), +the JPPS instance which keeps the job data must be contacted. +If it is known (\eg the only JPPS serving particular VO), the data retrieval +is straightforward using JPPS interfaces, as the jobid is the primary +key to access JPPS data. + +However, if JPPS for the job is not known, it must be looked up +using JPIS query. +Depending on the amount of the user's knowledge of the job details +\wrt\ JPIS configurations (\eg JPIS configured to request information +on jobs submitted in a~certain time interval is aware of the user's +job only if its submission time falls into this interval) +it may be necessary to query multiple JPIS's to find the particular job. + +\subsubsection{Job information retrieval} +Besides preserving the job data +the principal purpose of the \JP is to provide job information +according to some criteria, +freeing the user of the burden to keep complete records on her jobs. + +As discussed in Sect.~\ref{layered} the searches cannot be served +directly by the JPPS. +Therefore, the search must be done with querying a~particular JPIS +which configuration matches the user query: +\begin{itemize} +\item The user query criteria overlap with the JPIS configuration. +\Eg it makes little sense to look for +jobs submitted in May 2005 at a~JPIS restricted to be fed with data on +jobs submitted in 2006 only. + +\item The criteria trigger a~configured index at JPIS, avoiding +full scan through its data. +\end{itemize} +Again, it may be necessary to query multiple JPIS's and concatenate +the partial results. +Currently we do not address the potentially non-trivial problem of finding +suitable JPIS's. +It falls out of the scope of the \JP level, and should be preferably +solved at the service discovery level. + +\iffalse +Such a~search would result in scanning through all data stored in JP +which are expected to be huge, being unacceptable for frequent user queries. +Instead we define an architecture that allows +batch pre-processing of configurable queries. +The result of such query, a~superset of certain user query type, is further +indexed in order to provide fast response to concrete user queries. + + +We foreseen the following typical user queries: +\begin{itemize} +\item The user knows a jobid, job isn't longer in the \LB. He will ask the + JPPS to get all or selected attributes of job. +\item The user knows a jobid, job is in a terminal state. The user wants + all files (LB event dump, sandbox) stored by JP for further processing. +\item The user is looking for jobs with specific properties. In this case + (no jobid known) a JPIS must be used. There are the same query interface + provided by any JPIS but if a particular query can be answered by + the given JPIS depends on its configuration. + The user should know the proper JPIS to use for its particular + needs. +\item The user wants to add a user tag (annotation) to a job. He must know + the jobid(s). +\end{itemize} +\fi + + +\subsection{Security} + +The data stored in the \JP are in fact potentially more +sensitive as they also include information about the inputs and are kept +for eternity. All the interaction between components is +authenticated and only encrypted channels are used to transfer data. The +basic security model is inherited from the \LB, thus user and server +certificates are used for encryption and TLS is used for channel +encryption. + +The data in JPPS and JPIS are not encrypted as this would create a +problem with permanent depository of encryption keys. We also cannot use +users' public keys to encrypt the data as this would complicate sharing +and also endanger the data in case of private key loss. Instead, with a +very limited number of JPPS's deployed, we trust the \JP servers. Each +JPPS keeps list of authorized components (\LB, Resource Brokers,~\ldots) +that are allowed to upload data to the \JP server. + +The sensitive nature of data requires also strong authorization support. +While currently only implicit ACLs (only the job owner has access to +the data) are supported, we plan to use the VOMS +based authorization service to provide a fine grained (at the user/group +level) of authorization control. In the same way as in the \LB, users +will be able to specify who is authorized to access the data stored in +the \JP. In the current model, we plan to support read-only sharing, the +annotations should be always stored by the job owner only. However, a~way +to transfer ownership of the data to another person must be also +developed, to cover employees leave and even a death. + +\iffalse +\subsubsection{Internal and external interactions} + +Fig.~\ref{fig:psinter} shows interaction of \JP with other gLite components, +Fig.~\ref{fig:query} shows internal data flow in \JP as well as interaction +with the end-user (JP client). +In this section we discuss the involved operations and data transfers. +Unless specified otherwise, the communication occurs as web-service calls +over SSL-authenticated connections. +The interfaces are described in detail in~document ``EGEE Middleware Design'' +\footnote{https://edms.cern.ch/document/487871/}, +documented web service definitions can be found at \url{http://egee.cesnet.cz/en/WSDL/}. + +\emph{Search for jobs.} +The user does not known actual jobid's and searches for a~set of +jobs matching particular conditions (see Sect.~\ref{user}). +Such query cannot be served directly by \JP Primary Storage due to performance +reasons\Dash it would typically require sweep through a~large set of primary data. +On the contrary, the query must be directed to an Index Server that was +already populated with jobs matching a~broader condition. +By calling +the Index Server \texttt{QueryJobs} operation, the user +retrieves +a~set of jobid's with addresses of Primary Storage +where the data on the jobs are stored. + + +\subsubsection{Scalability and extensibility} +\todo{configuration, describe index server administrator role} +\todo{modularity (plug-ins), type plugin IS?} +\todo{najit kompromis pro rozdeleni informaci sem a do Service and + administrators view dole, nebo jedno zrusit} +\fi + diff --git a/org.glite.jp.doc/src/JPUG-Tools.tex b/org.glite.jp.doc/src/JPUG-Tools.tex new file mode 100644 index 0000000..11bfd34 --- /dev/null +++ b/org.glite.jp.doc/src/JPUG-Tools.tex @@ -0,0 +1,8 @@ +\section{Tools description} + +In this section we give a description of the tools that are installed +together with the \verb'glite-jp-client'. These are the only \JP\ tools that a +regular grid user may need to use. + +\input{jpimporter} +\input{gui} diff --git a/org.glite.jp.doc/src/JPUG-UseCases.tex b/org.glite.jp.doc/src/JPUG-UseCases.tex new file mode 100644 index 0000000..332e0dd --- /dev/null +++ b/org.glite.jp.doc/src/JPUG-UseCases.tex @@ -0,0 +1,3 @@ +\section{Use Cases} + +This section describes usage of \JP\ ... diff --git a/org.glite.jp.doc/src/JPUG.tex b/org.glite.jp.doc/src/JPUG.tex new file mode 100644 index 0000000..afa410f --- /dev/null +++ b/org.glite.jp.doc/src/JPUG.tex @@ -0,0 +1,43 @@ +\documentclass{egee} + +\input{definitions} + +\title{Job Provenance} +\Subtitle{User's Guide} +\author{CESNET EGEE II JRA1 team} +\DocIdentifier{EGEE-II....} +\Date{\today} +\Activity{JRA1: Middleware Engineering and Integration} +\DocStatus{DRAFT} +\Dissemination{PUBLIC} +\DocumentLink{http://...} + +\Abstract{This user's guide explains how to use the Job Provenance (\JP) +service from the user's point of view. The service architecture is described +thoroughly. Examples on using \JP\ are given... } + +\begin{document} + +\input{frontmatter} +\tableofcontents + +\newpage +\input{JPUG-Introduction} + +\newpage +\input{JPUG-Tools} + +\newpage +\input{JPUG-UseCases} + +%\appendix +%\newpage +%\input{JPUG-Appendix} + +\newpage +\nocite{jgc} +\bibliographystyle{unsrt} +\bibliography{lbjp} + +\end{document} + diff --git a/org.glite.jp.doc/src/copyright.tex b/org.glite.jp.doc/src/copyright.tex new file mode 100644 index 0000000..981ab62 --- /dev/null +++ b/org.glite.jp.doc/src/copyright.tex @@ -0,0 +1,24 @@ +% Taken from: +% https://twiki.cern.ch/twiki/bin/view/EGEE/EGEEgLiteSoftwareLicense +% +\vfill{} + +{\bf +Copyright} \copyright\ {\bf Members of the EGEE Collaboration. 2004. See +\href{http://www.eu-egee.org/partners/}{http://www.eu-egee.org/partners/} for +details on the copyright holders. + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + +\begin{center} +\href{http://www.apache.org/licenses/LICENSE-2.0}{http://www.apache.org/licenses/LICENSE-2.0} +\end{center} + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +} + diff --git a/org.glite.jp.doc/src/definitions.tex b/org.glite.jp.doc/src/definitions.tex new file mode 100644 index 0000000..34fcba3 --- /dev/null +++ b/org.glite.jp.doc/src/definitions.tex @@ -0,0 +1,17 @@ +\usepackage{xspace} +%\usepackage{doxygen} + +\def\LB{L\&B\xspace} +\def\JP{JP\xspace} +%\def\eg{e.\,g.} +\def\eg{for example\xspace} +\def\Eg{For example\xspace} +%\def\ie{i.\,e.} +\def\ie{that is\xspace} +\def\wrt{with respect to\xspace} +\def\Dash{---\penalty-1000} + +\long\def\TODO#1{\par\noindent\textbf{TODO:} {\sl#1}\par} +\long\def\ludek#1{} + +\hyphenation{plug-in} diff --git a/org.glite.jp.doc/src/egee.cls b/org.glite.jp.doc/src/egee.cls new file mode 100644 index 0000000..e0a03bf --- /dev/null +++ b/org.glite.jp.doc/src/egee.cls @@ -0,0 +1,507 @@ +% egee.cls: +% +% $Id$ +% +% $Log$ +% Revision 1.1.1.1 2007/11/13 17:42:10 jpospi +% initial import +% +% Revision 1.13 2004/08/31 19:24:27 szamsu +% Fixing overfull problem in page headers. Saving the logo and reusing it later, instead of loading in again. +% +% Revision 1.12 2004/08/09 14:03:54 szamsu +% proper IST number +% +% Revision 1.11 2004/08/03 17:02:21 szamsu +% Information Society Infrastrcutures logo replacing the old IST logo +% +% Revision 1.10 2004/08/03 13:34:37 szamsu +% Removed 'compat2' option on geometry, because it has also specified +% 'scale{0.8,0.9}' for page size, which we define otherwise. Added the +% 'centering' and 'includeheadfoot' options, which were defined in +% 'compat2'. +% +% Revision 1.9 2004/07/09 16:06:52 leanne +% Removed the Lead partner which is not used in egee +% +% Revision 1.8 2004/06/03 09:56:11 leanne +% removed lead partner + +% Revision 1.8 2004/06/03 09:56:11 diana +% updated the front page +% +% Revision 1.7 2004/05/26 09:38:55 leanne +% Removed DocumentLink in pdfinfo - it was causing errors +% +% Revision 1.6 2004/05/26 08:36:58 leanne +% Updated IST number in template +% +% Revision 1.5 2004/05/24 13:25:04 diana +% added template for egee latex documents +% +% Revision 1.4 2004/05/17 10:56:51 diana +% added compat2 option to geometry and hypertex option to hyperref to get logo and links back on the page +% +% Revision 1.3 2004/02/18 18:08:48 leanne +% Modified Document identifiers to mauch the EGEE Word templates +% +% Revision 1.2 2004/02/18 17:23:21 leanne +% Changed Work Package to Activity. Included definition of Document Link. +% +% Revision 1.1.1.1 2004/02/18 11:17:44 leanne +% Initial version of EGGE LaTeX Style files based on the EDG LaTeX Style +% +% +% Revision 1.0 2004/02/17 leanne +% Took the datagrid.cls file and modified it for the EGEE project +% Original Authors: +% Frohner Akos +% Diana Bosio +% Paul Millar +% Thanks are due to Norman Gray +% +\NeedsTeXFormat{LaTeX2e} +\ProvidesClass{egee}[2002/06/20 EGEE LaTeX Class] +\typeout{EGEE LaTeX class -- 2002/06/13 Rock Lobster!} +% +%% Notes: This class file tries, as largely as possible, to copy the Microsoft +%% Word template document EDMS 2098656 v2.2. Differences and notes are listed +%% below: +%% o The Word Template uses 11pt for the main body, but 12 point +%% occasionally. Any such occurrence of 12pt is mapped into 11pt in this +%% class-file. +%% o This class inherits 11pt article. In that class Huge=30pt and +%% LARGE=22pt, which matches the required point-size for the title page. +%% o The parskip in the Word doc is exactly 1.4mm (0.7mm above and below). +%% Here we've taken the liberty of adding some glue to make things fit +%% better. +%% o The Word Template shows all the (sub)sections on the contents page in +%% capitals and subsubsections in italics. The LateX class doesn't. + +%% Interface - example of an option, should we want to use these later. +%\newif\ifmonotitle\monotitlefalse + +%\DeclareOption{mono}{\monotitletrue} + +\DeclareOption*{\PassOptionsToClass{\CurrentOption}{article}} +\ProcessOptions + + +% Inherit! +\LoadClass[11pt]{article} + +% Necessary packages: +\RequirePackage{lastpage} +\RequirePackage{tabularx} +\RequirePackage{pslatex} +\RequirePackage{times} +\RequirePackage{verbatim} +\RequirePackage{geometry} +\RequirePackage{url} + +\usepackage[hang,bf,small]{caption} + +% +% We now define a new \if command to test for PDF being enabled. +% It is important because loading graphicx overrides the definition +% of \pdfoutput and sets it to true even when PDF is not enabled. +% Use \ifpdf instead of \ifx\pdfoutput\undefined hereafter. +% + +\newif\ifpdf +\ifx\pdfoutput\undefined + \pdffalse + % \typeout{PDF _not_ defined} +\else + \pdfoutput=1 + \pdftrue + % \typeout{PDF _is_ defined} +\fi + +\ifpdf + \usepackage[pdftex, + pdfpagemode={UseOutlines},bookmarks=true,bookmarksopen=true, + bookmarksopenlevel=0,bookmarksnumbered=true, + hypertexnames=false,colorlinks,linkcolor={blue}, + citecolor={blue},urlcolor={red}, + pdfstartview={FitV}]{hyperref} +\else + \usepackage[hypertex]{hyperref} +\fi + +\ifpdf + \usepackage[pdftex]{graphicx} + \pdfcompresslevel 9 + \pdfadjustspacing 1 +\else + \usepackage[dvips]{graphicx} +\fi + +\usepackage{color} + +\def\footsize{5mm} + +%% +%% PAGE GEOMETRY DEFINITIONS +%% +% From Template file +\geometry{centering,includeheadfoot} +\geometry{a4paper,top=12.5mm,headheight=12.5mm,headsep=5mm,foot=\footsize,footskip=13.3mm,bottom=12.5mm} +\geometry{right=25mm,left=25mm} + + +% APM -- I don't think these are right, my impression is above is correct +%\geometry{a4paper,margin=0.98in,headheight=0.72in} + + +%% +%% PAGE COLOUR DEFINITIONS +%% +\definecolor{blue}{rgb}{0.1,0.1,0.5} +\definecolor{lightgrey}{gray}{0.65} + + +% paulm's prefered name ... +\def\bibname{References} + +\setlength{\parindent}{0pt} +\setlength{\parskip}{1.4mm plus 0.4mm minus 0.2mm} + +\def\@defaultfooter{ + \def\@oddfoot{\vbox to \footsize {% + {\color{blue}\hrule width \textwidth height 1pt depth 0pt}% + \vfil + \small\hbox to \textwidth{\ISTNumber% + \hfil + \hbox{\colorbox{yellow}{\MakeUppercase{\@Dissemination}}}% + \hfil + \hbox{\thepage/\pageref{LastPage}}}% + }% + }% +} + + +\def\ps@title{% + \@defaultfooter + \def\@oddhead{\hbox to \textwidth{\LargeEGEELogo\hfil\ISTLogo}} +} + +\def\ps@headings{% + \@defaultfooter + \def\@oddhead{\vbox to \headheight{% +%\hrule width \textwidth height 1pt\relax + \vbox to 0.75\headheight{% + \hbox to \textwidth{% + \hbox to 0pt{\EGEELogo\hss}% + \hfil + \hbox to 8cm{% + \vbox to 0.75\headheight{% + \vfil + \parbox{8cm}{% + \centering\color{blue}% + \textbf{\MakeUppercase{\@title}}% +\ifx\@Subtitle\@empty\else + \par\textbf{\scriptsize\@Subtitle}% +\fi + }% + \vfil + }% + \hss}% + \hfil +%\hbox to 0pt{\vrule width 1pt height 10pt depth 0pt \hss}% +%% {\scriptsize\setlength{\parskip}{0pt}\setlength{\topsep}{0pt}% +%% % \vbox to 0.75\headheight{% +%% \parbox{4cm}{x% +%% \begin{flushright}% +%% \textit{Doc. Identifier}:\\ +%% \textbf{\@DocIdentifier}\\ +%% \vfil +%% \textit{Date}: \textbf{\@Date} +%% \end{flushright}% +%% }% +%% % }% +%% }% +\hbox to 0pt{\hss\vbox to 0.75\headheight{%\hrule +\tiny%\scriptsize +\parfillskip0pt +\leftskip 0pt plus 1fil +\parskip0ex +\textit{Doc.\ Identifier}: +\par +\textbf{\@DocIdentifier} +\vfil +\textit{Date}: \textbf{\@Date} +%\hrule +}}% +% \hbox to 4cm{\scriptsize +% \vbox to 0.75\headheight{% +% \parbox{4cm}{ +% \halign{\hfill####\cr +% \textit{Doc. Identifier}:\cr +% \textbf{\@DocIdentifier}\cr +% % \noalign{\vfil} +% \textit{Date}: \textbf{\@Date}\cr +% }}% +% \vfil +% }% +% }% + }% + }% +%\hrule width \textwidth height 1pt\relax + \vfil\vskip 2.5mm\relax + {\color{blue}\hrule width \textwidth height 1pt depth 0pt}% + }% + }% +} + +\pagestyle{headings} + +\setlength{\captionmargin}{1cm} + +% image file extensions respective to the output format +\ifpdf + \DeclareGraphicsExtensions{.jpg,.pdf,.png} + \pdfcompresslevel=9 +% \pdfinfo{ /Title (\@DocumentLink) } + \pdfinfo{ /Title (EGEE) } +\else + \DeclareGraphicsExtensions{.eps} +\fi + +\def\frontboxwidth{10.6cm}% + + + +%% +%% Define our title page +%% +\AtBeginDocument{ +\pagestyle{title}% +\hbox{}% Force top of page +\vfill +{\centering + \Huge\bf\textsf{\textcolor{blue}{EGEE}}\\[20mm]% + \LARGE\sc\textsf{\bf \@title}\\[5mm]% + \ifx\@Subtitle\@empty\else + \normalsize\textsf{\@Subtitle}\\[10mm]% + \fi + \ifx\@DeliverableId\@empty\else + \LARGE\sc\textsf{\bf \@DeliverableId}\\[5mm]% + \fi +}% +\vfill +\hbox to \textwidth{ + \hfil + \vbox{ + {\color{blue}\hrule width \frontboxwidth height 1mm depth 0pt} + \hbox to \frontboxwidth{\sf + \begin{tabularx}{\frontboxwidth}{l>{\raggedright\arraybackslash}X} + Document identifier: & \textbf{\@DocIdentifier}\\[3mm] + Date: & \textbf{\@Date}\\[3mm] + Activity:& \textbf{\@Activity}\\[3mm] + Document status: & \textbf{\@DocStatus}\\[3mm] + Document link:& \textbf{\@DocumentLink}\\[3mm] + \end{tabularx} + } + {\color{blue}\hrule width \frontboxwidth height 1mm depth 0pt} + } +} +\vfill +{\sf\underline{Abstract}: \@Abstract} +\vfill +\newpage % end of the first page +\pagestyle{headings} +\setcounter{tocdepth}{3} +} % End of AtBeginningDocument + + +% +% EGEE style small-capital section titles. +% +% The numbering is aligned with the WinWord style, +% although it is not common in the english typography... +% +\newcommand{\sectionbreak}{\newpage} +\renewcommand{\thesection}{\arabic{section}.} +\renewcommand{\thesubsection}{\thesection\arabic{subsection}.} +\renewcommand{\thesubsubsection}{\thesubsection\arabic{subsubsection}.} + +\renewcommand\section{\@startsection {section}{1}{\z@}% + {-3.5ex \@plus -1ex \@minus -.2ex}% + {2.3ex \@plus.2ex}% + {\normalfont\Large\bfseries\sffamily\scshape}} + +\renewcommand\subsection{\@startsection{subsection}{2}{\z@}% + {-3.25ex\@plus -1ex \@minus -.2ex}% + {1.5ex \@plus .2ex}% + {\normalfont\large\bfseries\sffamily\scshape}} +\renewcommand\subsubsection{\@startsection{subsubsection}{3}{\z@}% + {-3.25ex\@plus -1ex \@minus -.2ex}% + {1.5ex \@plus .2ex}% + {\normalfont\normalsize\bfseries\sffamily\scshape}} + + + +%% APM NEED TO REDEFINE section +%\titleformat{\section}{\Large\bfseries\sffamily\scshape}{\thesection}{1em}{} +%\titlecontents{section} [2em] {\vspace*{4pt}} +% {\large \sc \bfseries \contentslabel{2em}} +% {\large \sc \bfseries \hspace*{-2em}} +% {\large \textbf{\titlerule*[1ex]{.}\contentspage}} [\vspace*{4pt}] + +%\titleformat{\subsection}{\large\bfseries\sffamily\scshape}{\thesubsection}{1em}{} +%\titlecontents{subsection} [5em] {} +% {\sc \contentslabel{3em}} +% {\sc \hspace*{-3em}} +% {\titlerule*[1ex]{.}\contentspage} + + +% +% common constants +% +\def\ISTNumber{INFSO-RI-508833} +\newsavebox{\@EGEELogo} +\savebox{\@EGEELogo}{\includegraphics[height=0.75\headheight]{egee}} +\def\EGEELogo{\usebox{\@EGEELogo}} +\def\LargeEGEELogo{\includegraphics[height=\headheight]{egee}} +\def\ISTLogo{\includegraphics[height=\headheight]{isi}} + +% +% parameters to be supplied by the author +% +\def\Subtitle#1{\gdef\@Subtitle{#1}} +\gdef\@Subtitle{\@latex@warning@no@line{No \noexpand\Subtitle given}} + +\def\DeliverableId#1{\gdef\@DeliverableId{#1}} +\gdef\@DeliverableId{\@latex@warning@no@line{No \noexpand\DeliverableId given}} + +\def\DocIdentifier#1{\gdef\@DocIdentifier{#1}} +\gdef\@DocIdentifier{\@latex@warning@no@line{No \noexpand\DocIdentifier given % + (e.g. EGEE-JRA1-TEC-edmsId-v0-1)}} + +\def\Date#1{\gdef\@Date{#1}} +\gdef\@Date{\@latex@warning@no@line{No \noexpand\Date given % + (e.g. 01/01/2004)}} + +\def\Activity#1{\gdef\@Activity{#1}} +\gdef\@Activity{\@latex@warning@no@line{No \noexpand\Activity given % + (e.g. JRA1 Middleware Engineering and Integration )}} + +\def\DocStatus#1{\gdef\@DocStatus{#1}} +\gdef\@DocStatus{\@latex@warning@no@line{No \noexpand\DocStatus given % + (e.g. DRAFT, WORKING, DELIVERED)}} + +\def\Dissemination#1{\gdef\@Dissemination{#1}} +\gdef\@Dissemination{\@latex@warning@no@line{No \noexpand\Dissemination given % + (e.g. PUBLIC, INTERNAL, ...)}} + +\def\DocumentLink#1{\gdef\@DocumentLink{#1}} +\gdef\@DocumentLink{\@latex@warning@no@line{No \noexpand\DocumentLink given % + (e.g. http://cern.ch)}} + +\long\def\Abstract#1{\gdef\@Abstract{#1}} +\gdef\@Abstract{\@latex@warning@no@line{No \noexpand\Abstract given}} + +%% +%% Define the abstract using an environment abstract + +% +% This will produce the mailto link in the PDF file +% +% +% We use the URL package, which does this nicely. The old way (\HTTP) was +% a bit buggy as it had problems with '~'s and '_'s +% +\urlstyle{sf} +\ifpdf + \newcommand{\Email}[1]{\href{mailto:#1}{<{#1}>}} + \newcommand{\HTTP}[1]{\href{#1}{\url{#1}}} +\else + \newcommand{\Email}[1]{\textsf{<{#1}>}} + \newcommand{\HTTP}[1]{\url{#1}} +\fi + + +% +% We now redifine \part and \section so that the table of contents +% has the sections/parts in upper case. +% +% Note: need to use \uppercase because \MakeUppercase is not robust +% +\def\@part[#1]#2{% + \ifnum \c@secnumdepth >\m@ne + \refstepcounter{part}% + \addcontentsline{toc}{part}{\thepart\hspace{1em}\uppercase{#1}}% + \else + \addcontentsline{toc}{part}{\uppercase{#1}}% + \fi + {\parindent \z@ \raggedright + \interlinepenalty \@M + \normalfont + \ifnum \c@secnumdepth >\m@ne + \Large\bfseries \partname\nobreakspace\thepart + \par\nobreak + \fi + \huge \bfseries #2% + \markboth{}{}\par}% + \nobreak + \vskip 3ex + \@afterheading} + +\def\@sect#1#2#3#4#5#6[#7]#8{% + \ifnum #2>\c@secnumdepth + \let\@svsec\@empty + \else + \refstepcounter{#1}% + \protected@edef\@svsec{\@seccntformat{#1}\relax}% + \fi + \@tempskipa #5\relax + \ifdim \@tempskipa>\z@ + \begingroup + #6{% + \@hangfrom{\hskip #3\relax\@svsec}% + \interlinepenalty \@M #8\@@par}% + \endgroup + \csname #1mark\endcsname{\uppercase{#7}}% + \addcontentsline{toc}{#1}{% + \ifnum #2>\c@secnumdepth \else + \protect\numberline{\csname the#1\endcsname}% + \fi + \texorpdfstring{\uppercase{#7}}{#7}}% + \else + \def\@svsechd{% + #6{\hskip #3\relax + \@svsec #8}% + \csname #1mark\endcsname{\uppercase{#7}}% + \addcontentsline{toc}{#1}{% + \ifnum #2>\c@secnumdepth \else + \protect\numberline{\csname the#1\endcsname}% + \fi + \texorpdfstring{\uppercase{#7}}{#7}}}% + \fi + \@xsect{#5}} + +% \addcontentsline{toc} expands to \contentsline{NAME} +% which in turn expands to \l@NAME. So, to specify +% the table of contents, we must define \l@chapter, \l@section, +% \l@subsection, ... ; to specify the list of figures, we must define +% \l@figure; and so on. Most of these can be defined with the +% \@dottedtocline command, which produces a contents line with dots +% between the title and the page number. It works as follows: +% +% \@dottedtocline{LEVEL}{INDENT}{NUMWIDTH} +% LEVEL : An entry is produced only if LEVEL < or = value of +% 'tocdepth' counter. Note, \chapter is level 0, \section +% is level 1, etc. +% INDENT : The indentation from the outer left margin of the start of +% the contents line. +% NUMWIDTH : The width of a box in which the section number is to go, +% if TITLE includes a \numberline command. +% + +\def\l@part{\@dottedtocline{1}{4em}{2.0em}} +\def\l@subsection{\@dottedtocline{2}{1.5em}{2.3em}} +\def\l@subsubsection{\@dottedtocline{3}{3.8em}{3.2em}} +\def\l@paragraph{\@dottedtocline{4}{7.0em}{4.1em}} +\def\l@subparagraph{\@dottedtocline{5}{10em}{5em}} + diff --git a/org.glite.jp.doc/src/frontmatter.tex b/org.glite.jp.doc/src/frontmatter.tex new file mode 100644 index 0000000..732dcec --- /dev/null +++ b/org.glite.jp.doc/src/frontmatter.tex @@ -0,0 +1,40 @@ +\begin{center} +{\bf Delivery Slip} +\end{center} +\begin{tabularx}{\textwidth}{|l|l|l|X|X|} +\hline + & {\bf Name} & {\bf Partner} & {\bf Date} & {\bf Signature} \\ +\hline +{\bf From} & & & & \\ +\hline +{\bf Reviewed by} & & & & \\ + +\hline +{\bf Approved by} & & & & \\ +\hline +\end{tabularx} + +\begin{center} +{\bf Document Change Log} +\end{center} + +\begin{tabularx}{\textwidth}{|l|l|X|X|} +\hline +{\bf Issue } & {\bf Date } & {\bf Comment } & {\bf Author } \\ \hline + +\hline +\end{tabularx} + +\begin{center} +{\bf Document Change Record} +\end{center} + +\begin{tabularx}{\textwidth}{|l|l|X|} +\hline +{\bf Issue } & {\bf Item } & {\bf Reason for Change } \\ \hline + +\hline +\end{tabularx} + +\input{copyright} +\clearpage diff --git a/org.glite.jp.doc/src/gui.tex b/org.glite.jp.doc/src/gui.tex new file mode 100644 index 0000000..dae3aec --- /dev/null +++ b/org.glite.jp.doc/src/gui.tex @@ -0,0 +1,3 @@ +\subsection{GUI} + +\TODO{GUI description} diff --git a/org.glite.jp.doc/src/jpimporter.tex b/org.glite.jp.doc/src/jpimporter.tex new file mode 100644 index 0000000..3169ba6 --- /dev/null +++ b/org.glite.jp.doc/src/jpimporter.tex @@ -0,0 +1,2 @@ +\subsection{glite-jp-importer} + diff --git a/org.glite.jp.doc/src/lbjp.bib b/org.glite.jp.doc/src/lbjp.bib new file mode 100644 index 0000000..dd164b4 --- /dev/null +++ b/org.glite.jp.doc/src/lbjp.bib @@ -0,0 +1,709 @@ + +@InProceedings{ condor, + author = "Michael Litzkow and Miron Livny and Matthew Mutka", + title = "{C}ondor - A Hunter of Idle Workstations", + booktitle = "Proceedings of the 8th International Conference of + Distributed Computing Systems", + year = "1988", + month = "June" +} + +@InProceedings{ gram, + author = "K. Czajkowski and I. Foster and N. Karonis and C. + Kesselman and Martin S. Smith and S. Tuecke", + title = "A resource management architecture for metacomputing + systems.", + booktitle = "Proceedings of the IPPS/SPDP Workshop on Job Scheduling + Strategies for Parallel Processing", + year = "1988", + pages = "62-82" +} + +@InProceedings{ lsf, + author = "S. Zhou", + title = "LSF: Load sharing in large-scale heterogenous distributed + systems", + booktitle = "Proceedings of the Workshop on Cluster Computing", + year = "1992" +} + +@Misc{ dce, + author = "Open Group CAE Specification", + title = "DCE: Remote Procedure Call", + howpublished = "ISBN 1-85912-041-5", + year = "1994" +} + +@Misc{ pbs, + author = "Henderson, R. and Tweten, D", + title = "Portable Batch System: External reference Specification", + howpublished = "NASA, Ames Research Center", + year = "1996" +} + +@Article{ globus, + author = "I. Foster and C. Kesselman", + title = "Globus: a metacomputing infrastructure toolkit.", + journal = "International Journal of Supercomputer Applications", + year = "1997", + volume = "11", + number = "2", + pages = "115-128" +} + +@Misc{ ulm, + author = "J. Abela and T. Debeaupuis", + title = "Universal Format for Logger Messages", + howpublished = "IETF Internet Draft", + year = "1997" +} + +@InProceedings{ autopilot, + author = "R.L. Ribler and J.S. Vetter and H. Simitci and D.A. Reed", + title = "Autopilot: adaptive control of distributed applications", + booktitle = "Proceedings of the Seventh IEEE Symposium on + High-Performance Distributed Computing", + year = "1998", + pages = "172-179" +} + +@InProceedings{ gsi, + author = "I. Foster and C. Kesselman and G. Tsudik and S. Tuecke", + title = "{A Security Architecture for Computational Grids}", + booktitle = "Proceedings of the 5th ACM Conference on Computer and + Communications Security Conference", + year = 1998, + pages = "83-92" +} + +@InProceedings{ classad, + author = "Rajesh Raman and Miron Livny and Marvin Solomon", + title = "Matchmaking: Distributed Resource Management for High + Throughput Computing", + booktitle = "Proceedings of the Seventh IEEE International Symposium on + High Performance Distributed Computing", + year = "1998" +} + +@Article{ nws, + author = "R.Wolski and N. Spring and J. Hayes", + title = "The network weather service: a distributed resource + performance forecasting service for metacomputing", + journal = "J. Future Generation Comput. Syst.", + year = "1999", + volume = "15", + number = "(5/6)", + pages = "757-768" +} + +@Misc{ tls, + author = "T. Dierks and C. Allen", + title = "The {TLS} {P}rotocol {V}ersion 1.0", + howpublished = "IETF RFC 2246 (Standards Track)", + year = 1999, + month = "January" +} + +@Misc{ gssapi, + author = "{J. Linn}", + title = "{Generic Security Service Application Program Interface + Version 2, Update 1}", + howpublished = "IETF RFC 2743", + year = "2000", + month = "January" +} + +@TechReport{ lbdraft, + author = {M. Ruda and A. Krenek and L. Matyska and J. Pospisil and + M. Vocu and J. Sitera}, + title = {Logging and bookkeeping architecture}, + institution = {EU DataGrid}, + year = {2001}, + note = {Part of Deliverable D1.1 WP1 Report on current + technology} +} + +@InProceedings{ mds2, + author = "K. Czajkowski and S. Fitzgerald and I. Foster and C. + Kesselman.", + title = "Grid Information Services for Distributed Resource + Sharing.", + booktitle = " Proceedings of the Tenth IEEE International Symposium on + High-Performance Distributed Computing (HPDC-10)", + year = "2001", + month = "August", + publisher = "IEEE Press" +} + +@Misc{ gma, + author = "B. Tierney and R. Aydt and D. Gunter and W. Smith and + Valerie Taylor and R. Wolski and M. Swany.", + title = "A Grid Monitoring Service Architecture.", + howpublished = "Global Grid Forum Performance Working Group", + year = "2001" +} + +@Misc{ rgma, + author = "S. Fisher", + title = "Relational Model for Information and Monitoring", + howpublished = "Technical Report GWD-Perf-7-1, GGF", + year = "2001" +} + +@InProceedings{ mercury, + author = "Zoltan Balaton and Peter Kacsuk and Norbert Podhorszki and + Ferenc Vajda.", + title = "From Cluster Monitoring to Grid Monitoring Based on GRM", + booktitle = "In proceedings 7th EuroPar2001 Parallel Processings, + Manchester, UK.", + year = "2001", + pages = "874-881" +} + +@Article{ siena, + author = "Antonio Carzaniga and others", + title = "Design and evaluation of a wide-area event notification + service", + journal = "ACM Transactions on Computer Systems", + year = "2001", + volume = "19", + number = "3", + pages = "332-383" +} + +@InProceedings{ maui, + author = "D. Jackson and Q. Snell and M. Clement", + title = "Core algorithms of the Maui scheduler", + booktitle = "Proceedings of the 7th Workshop on Job Scheduling + Strategies for Parallel Processing", + year = "2001" +} + +@Misc{ ggfxml, + author = "W. Smith and D. Gunther and D. Quesnel", + title = "A simple xml producer-consumer protocol", + howpublished = "Grid Working Document GWD-Perf-8-2, Global Grid Forum, + Performance Working Group", + year = "2001" +} + +@TechReport{ lbdraft2, + author = {M. Ruda and A. Krenek and L. Matyska and J. Pospisil and + M. Vocu and J. Sitera}, + title = {Logging and bookkeeping architecture for {D}ata{G}rid + {R}elease 2}, + institution = {EU DataGrid}, + year = {2002}, + note = {Part of Deliverable D1.2 Definition of architecture, + technical plan and evaluation criteria for scheduling, + resource management, security and job description} +} + +@InProceedings{ code, + author = "W. Smith", + title = "A System for Monitoring and Management of Computational + Grids.", + booktitle = "In Proceedings of the 2002 International Conference on + Parallel Processing", + year = "2002" +} + +@Misc{ hawkeye, + title = "HawkEye: A Monitoring and Management Tool for Distributed + Systems", + howpublished = "\url{http://www.cs.wisc.edu/condor/hawkeye}", + year = "2002" +} + +@InProceedings{ chep03, + author = {G. Avellino et.al.}, + title = {{The first deployment of workload management services on + the EU DataGrid testbed: feedback on design and + implementation}}, + booktitle = {Computing in High Energy and Nuclear Physics (CHEP03)}, + year = {2003}, + note = {La Jolla, Ca, USA, March 2003}, + fauthor = {G. Avellino, S. Beco, B. Cantalupo, F. Pacini, A. + Terracina, A. Maraschini, D. Colling, S. Monforte, M. + Pappalardo, L. Salconi, F. Giacomini, E. Ronchieri, D. + Kouril, A. Krenek, L. Matyska, M. Mulac, J. Pospisil, M. + Ruda, Z. Salvet, J. Sitera, M. Vocu, M. Mezzadri, F. Prelz, + A. Gianelle, R. Peluso, M. Sgaravatto, S. Barale, A. + Guarise, A. Werbrouck} +} + +@Article{ alien, + author = "P. Saiz and others", + title = "{AliEn - ALICE environment on the GRID}", + journal = "{Nuclear Instruments and Methods in Physics Research, + Section A}", + year = "2003", + volume = "502", + number = "2-3", + pages = "437-440", + fauthor = { P. Saiz, L. Aphecetche, P. Buncic, R. Piskac, J.E. + Revsbech, V. Sego} +} + +@InProceedings{ rgma2, + author = "A. Cooke and others", + title = "R-GMA: An information integration system for grid + monitoring", + booktitle = "Proc. of the 11th International Conference on Cooperative + Information Systems", + year = "2003" +} + +@InProceedings{ netlogger, + author = "B. Tierney and D. Gunter", + title = "NetLogger: a toolkit for distributed system performance + tuning and debugging", + booktitle = "Proceedings of the IFIP/IEEE Eighth International + Symposium on Integrated Network Management (IM 2003)", + year = "2003", + pages = "97-100", + publisher = "Kluwer", + bolume = "246 of IFIP Conference Proceedings" +} + +@InProceedings{ monalisa, + author = "H.B. Newman and I.C. Legrand and P. Galvez and R. Voicu + and C. Cirstoiu", + title = "MonALISA: a distributed monitoring service architecture", + booktitle = "Computing in High Energy and Nuclear Physics (CHEP03), La + Jolla, CA", + year = "2003" +} + +@InProceedings{ gridrm, + author = "M.A. Baker and G.C. Smith", + title = "GridRM: an extensible resource monitoring system", + booktitle = "Proceedings of the IEEE International Cluster Computing + Conference", + year = "2003", + pages = "207-214" +} + +@InProceedings{ gridlab, + author = "Zoltan Balaton and Gabor Gombas", + title = "Resource and Job Monitoring in the Grid", + booktitle = "Proc. of the Euro-Par 2003 International Conference, + Klagenfurt", + year = 2003 +} + +@Misc{ rgmacms, + author = "D. Bonacorsi and D. Colling and L. Field and S. Fisher and + C. Grandi and P. R. Hobson and P. Kyberd and B. MacEvoy and + J. J. Nebrensky and H. Tallini and S. Traylen", + title = "Scalability Tests of R-GMA Based Grid Job Monitoring + System for CMS Monte Carlo Data Production", + howpublished = "IEEE NSS Conference, Oregon, USA", + year = "2003", + month = "21-24 October" +} + +@Misc{ xacml, + author = "{OASIS Standard}", + title = "{eXtensible Access Control Markup Language (XACML), + Version 1.0}", + year = 2003, + month = "February" +} + +@InProceedings{ gacl1, + author = "L. Cornwall and J. Jensen and D. Kelsey and A. McNab", + title = "{EU DataGrid and GridPP authorization and access control}", + booktitle = "Proceedings of the UK e-Science All Hands Meeting", + year = "2003", + pages = "382--384" +} + +@Misc{ glue, + author = "S. Andreozzi and M. Sgaravatto and C. Vistoli", + title = "Sharing a conceptual model of Grid resources and services", + howpublished = "Proceedings of the Conference for Computing in High Energy + and Nuclear Physics (CHEP03)", + year = "2003" +} + +@InProceedings{ perf, + author = "Xuehai Zhang and Jeffrey L. Freschl and Jennifer M. + Schopf", + title = "A Performance Study of Monitoring and Information Services + for Distributed Systems", + booktitle = "12th IEEE International Symposium on High Performance + Distributed Computing (HPDC-12 '03)", + year = "2003" +} + +@InProceedings{ glite, + author = "E. Laure and F. Hemmer and F. Prelz and S. Beco and S. + Fisher and M. Livny and L. Guy and M. Barroso and P. Buncic + and P. Kunszt and A. {Di Meglio} and A. Aimar and A. Edlund + and D. Groep and F. Pacini and M. Sgaravatto and O. Mulmo", + title = "Middleware for the next generation Grid infrastructure", + booktitle = "Computing in High Energy Physics and Nuclear Physics (CHEP + 2004)", + year = 2004 +} + +@Article{ jgc, + author = {G. Avellino and others}, + title = {{The DataGrid Workload Management System: Challenges and + Results}}, + journal = {Journal of Grid Computing}, + year = {Dec 2004}, + volume = {2}, + number = {4}, + pages = {353--367}, + fauthor = { G. Avellino, S. Beco, B. Cantalupo, A. Maraschini, F. + Pacini, M. Sottilaro, A. Terracina, D. Colling, F. + Giacomini, E. Ronchieri, A. Gianelle, M. Mazzucato, R. + Peluso, M. Sgaravatto, A. Guarise, R. Piro, A. Werbrouck, + D. Kouøil, A. Køenek, L. Matyska, M. Mulaè, J. Pospí¹il, M. + Ruda, Z. Salvet, J. Sitera, J. ©krabal, M. Vocù, M. + Mezzadri, F. Prelz, S. Monforte, M. Pappalardo} +} + +@Misc{ apart, + author = "M. Gerndt and others", + title = "Performance Tools for the {Grid}: State of the Art and + Future", + howpublished = "Tech. Rep. Lehrstuhl fuer Rechnertechnik und + Rechnerorganisation, Technische Universitaet Muenchen + (LRR-TUM). + \url{http://www.lpds.sztaki.hu/~zsnemeth/apart/}", + year = "2004" +} + +@InProceedings{ ahm04194, + author = {Shrija Rajbhandari and David W. Walker}, + title = "{Support for Provenance in a Service-based Computing + Grid}", + booktitle = "{Proceedings of the third UK e-Science All Hands Meeting, + Nottingham, UK}", + year = {2004} +} + +@TechReport{ gro04, + author = {Paul T. Groth}, + title = "{Recording provenance in service-oriented architectures}", + institution = "{University of Southampton; Faculty of Engineering, + Science and Mathematics; School of Electronics and Computer + Science}", + year = {2004} +} + +@InProceedings{ glm04b, + author = {Paul Groth and Michael Luck and Luc Moreau}, + title = "{A protocol for recording provenance in service-oriented + grids}", + booktitle = "{Proceedings of the 8th International Conference on + Principles of Distributed Systems (OPODIS04)}", + year = {2004} +} + +@Article{ ganglia, + author = "M.L. Massie and B.N. Chun and D.E. Culler", + title = "Ganglia Distributed Monitoring System: Design, + Implementation, and Experience", + journal = "Parallel Computing", + year = "2004", + volume = "30", + pages = "817-840" +} + +@Misc{ gae, + author = "Arshad Ali and Ashiq Anjum and Julian Bunn and Richard + Cavanaugh and Frank van Lingen and Richard McClatchey and + Harvey Newman and Waqas ur Rehman and Conrad Steenberg and + Michael Thomas and Ian Willers", + title = "Job Monitoring in an Interactive Grid Analysis + Environment", + howpublished = "Computing in High Energy Physics 2004", + year = "2004" +} + +@InProceedings{ voms1, + author = "R. Alfieri and R. Cecchini and V. Ciaschini and L. + dell'Agnello and \'{A}. Frohner and A. Gianoli and K. + L\H{o}rentey and F. Spataro", + title = "{VOMS}, an {A}uthorization {S}ystem for {V}irtual + {O}rganizations", + booktitle = "Grid Computing: First European Across Grids Conference", + year = "2004" +} + +@InProceedings{ delegation, + author = "V. Welch and I. Foster and C. Kesselman and O. Mulmo and + L. Pearlman and S. Tuecke and J. Gawor and S. Meder and F. + Siebenlist", + title = "{X.509 Proxy Certificates for Dynamic Delegation}", + booktitle = "Proceedings of the 3rd Annual PKI R\&D Workshop", + year = 2004, + month = "April" +} + +@Misc{ proxycert, + author = "S. Tuecke and V. Welch and D. Engert and L. Pearlman and + M. Thompson", + title = "Internet {X}.509 {P}ublic {K}ey {I}nfrastructure ({PKI}) + proxy certificate profile", + howpublished = "IETF RFC 3820", + year = 2004, + month = "June" +} + +@InProceedings{ chep, + author = "D. Kouøil and others", + title = "Distributed Tracking, Storage, and Re-use of Job State + Information on the Grid", + booktitle = "Computing in High Energy and Nuclear Physics (CHEP04)", + year = "2004" +} + +@Misc{ genericlb, + author = "{CESNET JRA1 team}", + title = "Current and Foreseen {\LB}\ Design", + howpublished = "3rd EGEE Conference, Athens, + \url{http://indico.cern.ch/contributionDisplay.py?contribId=201&sessionId=73&confId=0513}" + , + year = 2005 +} + +@InProceedings{ eup1, + author = "Liming Chen and Victor Tan and Fenglian Xu and Alexis + Biller and Paul Groth and Simon Miles and John Ibbotson and + Michael Luck and Luc Moreau", + title = "{A proof of concept: Provenance in a Service Oriented + Architecture}", + booktitle = "{Proceedings of the fourth UK e-Science All Hands Meeting, + Nottingham, UK}", + year = {2005} +} + +@InProceedings{ tgx05, + author = {Paul Townend and Paul Groth and Jie Xu}, + title = "{A provenance-aware weighted fault tolerance scheme for + service-based applications}", + booktitle = "{Proceedings of the $8^{th}$ IEEE International Symposium + on Object-oriented Real-time distributed Computing (ISORC + 2005)}", + year = {2005} +} + +@InProceedings{ mygrid, + author = "M. Nedim Alpdemir and Arijit Mukherjee and Norman W. Paton + and Alvaro A.A. Fernandes and Paul Watson and Kevin Glover + and Chris Greenhalgh and Tom Oinn and Hannah Tipney", + title = "Contextualised Workflow Execution in myGrid", + booktitle = " Proc European Grid Conference , Springer-Verlag LNCS + 3470", + year = 2005, + pages = "444-453" +} + +@Article{ condor2, + author = "Douglas Thain and Todd Tannenbaum and Miron Livny", + title = "Distributed computing in practice: the Condor experience.", + journal = "Concurrency - Practice and Experience", + year = "2005", + volume = "17", + number = "2-4", + pages = "323-356" +} + +@InProceedings{ globus4, + author = "I. Foster", + title = "Globus Toolkit Version 4: Software for Service-Oriented + Systems.", + booktitle = "IFIP International Conference on Network and Parallel + Computing, Springer-Verlag LNCS 3779,", + year = "2005", + pages = "2-13" +} + +@InProceedings{ cgmapisa, + author = "Andrea Ceccanti and Ond\v{r}ej Kraj\'{\i}\v{c}ek and + Ale\v{s} K\v{r}enek and Lud\v{e}k Matyska and Miroslav + Ruda.", + title = "Towards Scalable and Interoperable Grid Monitoring + Infrastructure.", + booktitle = "Proceedings of the first CoreGRID Integration Workshop", + year = "2005", + pages = "10-18" +} + +@InProceedings{ cgma, + author = "Ondøej Krajíèek and Ale¹ Køenek and Ludìk Matyska and + Miroslav Ruda and Jiøí Sitera", + title = "Capability Languages in C-GMA", + booktitle = "Proceedings of Cracow Grid Workshop (CGW05)", + year = "2005" +} + +@Article{ taxonomy, + author = {Serafeim Zanikolas and Rizos Sakellariou}, + title = {A taxonomy of grid monitoring systems}, + journal = {Future Gener. Comput. Syst.}, + year = {2005}, + volume = {21}, + number = {1}, + pages = {163--188}, + issn = {0167-739X}, + doi = {http://dx.doi.org/10.1016/j.future.2004.07.002}, + publisher = {Elsevier Science Publishers B. V.}, + address = {Amsterdam, The Netherlands, The Netherlands} +} + +@Article{ unicore, + author = "Achim Streit and Dietmar Erwin and Thomas Lippert and + Daniel Mallmann and Roger Menday and Michael Rambadt and + Morris Riedel and Mathilde Romberg and Bernd Schuller and + Philipp Wieder", + title = "UNICORE - From Project Results to Production Grids", + journal = "Grid Computing : New Frontiers of High Performance + Computing", + year = "2005", + pages = "357 - 376" +} + +@Article{ voms2, + author = "R. Alfieri and R. Cecchini and V. Ciaschini and L. + dell'Agnello and \'{A}. Frohner and K. L\H{o}rentey and F. + Spataro", + title = "{From gridmap-file to VOMS: managing authorization in a + Grid environment}", + journal = "Future Generation Computer Systems", + year = "2005", + pages = "549-558", + month = "April" +} + +@InProceedings{ cgw05, + author = "F. Dvoøák and D. Kouøil and A. Køenek and L. Matyska and + M. Mulaè and J.Pospi¹il and M. Ruda and Z. Salvet and J. + Sitera and J. ©krabal and M. Vocù ", + title = "Services for Tracking and Archival of Grid Job + Information", + booktitle = "Proceeding of Cracow Grid Workshop", + year = "2005" +} + +@Misc{ rfc4122, + author = "P. Leach and M. Mealling and R. Salz", + title = "A Universally Unique IDentifier (UUID) URN Namespace", + howpublished = "IETF RFC 4122", + year = "2005" +} + +@Misc{ euprovenance, + title = "{EU FP6 Programme Enabling and Supporting Provenance in + Grids for Complex Problems}", + howpublished = "\url{http://twiki.gridprovenance.org/bin/view/Provenance/ProjectInformation}" + , + year = 2006 +} + +@Misc{ atlas, + author = "TODO - predelat na neco poradneho", + title = "{The ATLAS Experiment}", + howpublished = "\url{http://atlas.ch/}", + year = 2006 +} + +@Misc{ ami, + author = "TODO - predelat na neco poradneho", + title = "{The Atlas Metadata Interface}", + howpublished = "\url{https://atlastagcollector.in2p3.fr:8443/AMI/}", + year = 2006 +} + +@Misc{ alice, + author = "TODO - predelat na neco poradneho", + title = "{A Large Ion Collider Experiment}", + howpublished = "\url{http://aliceinfo.cern.ch/Public}", + year = 2006 +} + +@Misc{ sam, + author = "TODO - predelat na neco poradneho", + title = "{SAM (Sequential data Access via Meta-data)}", + howpublished = "\url{http://d0db.fnal.gov/sam/}", + year = 2006 +} + +@InProceedings{ gacl2, + author = "A. McNab and S. Kaushal", + title = "Web services with gridsite and C/C++/scripts", + booktitle = "Computing in High Energy and Nuclear Physics (CHEP 2006)", + year = "2006" +} + +@Misc{ crossgrid, + author = "CrossGrid", + title = "{CrossGrid Project}", + howpublished = "\url{http://www.crossgrid.org}" +} + +@Misc{ dgas, + author = "DGAS", + title = "{The Distributed Grid Accounting System}", + howpublished = "\url{http://www.to.infn.it/grid/accounting/main.html}" +} + +@Misc{ egee, + author = "EGEE", + title = "{Enabling Grids for E-sciencE (EGEE) Project}", + howpublished = "\url{http://lcg.web.cern.ch/LCG/}" +} + +@Misc{ gridice, + author = "EGEE", + title = "{GridIce, the eyes of the Grid}", + howpublished = "\url{http://gridice2.cnaf.infn.it:50080/gridice/}" +} + +@Misc{ djra1.2, + author = {{EGEE JRA1}}, + title = {{EGEE Middleware Design---Release 1}}, + howpublished = {\url{https://edms.cern.ch/document/487871/}} +} + +@Misc{ djra1.3, + author = {{EGEE JRA1}}, + title = {{EGEE Middleware Release 1}}, + howpublished = {\url{https://edms.cern.ch/document/567624/}} +} + +@Misc{ djra1.4, + author = {{EGEE JRA1}}, + title = {{EGEE Middleware Architecture---Release 2}}, + howpublished = {\url{https://edms.cern.ch/document/594698/}} +} + +@Misc{ jra2jobmetrics, + author = "EGEE JRA2", + title = "{Job Metrics}", + howpublished = "\url{http://egee-jra2.web.cern.ch/EGEE-JRA2/QoS/JobsMetrics/JobMetrics.htm}" + +} + +@Misc{ lbug, + author = "A. K\v{r}enek and others", + title = "{L\&B Users Guide}", + howpublished = "\url{https://edms.cern.ch/file/571273/1/LB-guide.pdf}" +} + +@Misc{ lcg, + author = "LCG", + title = "{LHC Computing Project (LCG)}", + howpublished = "\url{http://lcg.web.cern.ch/LCG/}" +} + +@Misc{ pasoa, + title = "{PASOA: Provenance Aware Service Oriented Architecture}", + howpublished = "\url{http://twiki.pasoa.ecs.soton.ac.uk/bin/view/PASOA/AboutPasoa}" + +} -- 1.8.2.3