%APN3_PROCEEDINGS_FORM%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% TEMPLATE.TEX -- APN3 (2003) ASP Conference Proceedings template.
%
% Derived from ADASS VIII (98) ASP Conference Proceedings template
% Updated by N. Manset for ADASS IX (99), F. Primini for ADASS 2000,
% D.Bohlender for ADASS 2001, and H. Payne for ADASS XII and LaTeX2e.
%
% Use this template to create your proceedings paper in LaTeX format
% by following the instructions given below.  Much of the input will
% be enclosed by braces (i.e., { }).  The percent sign, "%", denotes
% the start of a comment; text after it will be ignored by LaTeX.  
% You might also notice in some of the examples below the use of "\ "
% after a period; this prevents LaTeX from interpreting the period as
% the end of a sentence and putting extra space after it.  
% 
% You should check your paper by processing it with LaTeX.  For
% details about how to run LaTeX as well as how to print out the User
% Guide, consult the README file.  You should also consult the sample
% LaTeX papers, sample1.tex and sample2.tex, for examples of including
% figures, html links, special symbols, and other advanced features.
%
% If you do not have access to the LaTeX software or a laser printer
% at your site, you can still prepare your paper following the
% instructions in the User Guide.  In such cases, the editors will
% process the file and make any necessary editorial adjustments.
% 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% 
\documentclass[11pt,twoside]{article}  % Leave intact
\usepackage{adassconf}

% If you have the old LaTeX 2.09, and not the current LaTeX2e, comment
% out the \documentclass and \usepackage lines above and uncomment
% the following:

%\documentstyle[11pt,twoside,adassconf]{article}

\begin{document}   % Leave intact

%-----------------------------------------------------------------------
%			    Paper ID Code
%-----------------------------------------------------------------------
% Enter the proper paper identification code.  The ID code for your
% paper is the session number associated with your presentation as
% published in the official conference proceedings.  You can           
% find this number locating your abstract in the printed proceedings
% that you received at the meeting or on-line at the conference web
% site; the ID code is the letter/number sequence proceeding the title 
% of your presentation. 
%
% This will not appear in your paper; however, it allows different
% papers in the proceedings to cross-reference each other.  Note that
% you should only have one \paperID, and it should not include a
% trailing period.
%
% EXAMPLE: \paperID{O4-1}
% EXAMPLE: \paperID{P7-7}
%

\paperID{O10-3}
%%%% ID=O10-3

%-----------------------------------------------------------------------
%		            Paper Title 
%-----------------------------------------------------------------------
% Enter the title of the paper.
%
% EXAMPLE: \title{A Breakthrough in Astronomical Software Development}
% 
% If your title is so long as to fill the page header when you print it,
% then please supply a short form as a \titlemark.
%
% EXAMPLE: 
%  \title{Rapid Development for Distributed Computing, with Implications
%         for the Virtual Observatory}
%  \titlemark{Rapid Development for Distributed Computing}
%

\title{VLT Instruments Pipeline System Overview}
%\titlemark{ }

%-----------------------------------------------------------------------
%		          Authors of Paper
%-----------------------------------------------------------------------
% Enter the authors followed by their affiliations.  The \author and
% \affil commands may appear multiple times as necessary (see example
% below).  List each author by giving the first name or initials first
% followed by the last name.  Authors with the same affiliations
% should grouped together. 
%
% EXAMPLE: \author{Raymond Plante, Doug Roberts, 
%                  R.\ M.\ Crutcher\altaffilmark{1}}
%          \affil{National Center for Supercomputing Applications, 
%                 University of Illinois Urbana-Champaign, Urbana, IL
%                 61801}
%          \author{Tom Troland}
%          \affil{University of Kentucky}
%
%          \altaffiltext{1}{Astronomy Department, UIUC}
%
% In this example, the first three authors, "Plante", "Roberts", and
% "Crutcher" are affiliated with "NCSA".  "Crutcher" has an alternate 
% affiliation with the "Astronomy Department".  The fourth author,
% "Troland", is affiliated with "University of Kentucky"

\author{Yves Jung, Pascal Ballester, Klaus Banse, Wolfgang Hummel, Carlo
Izzo, Lars K. Lundin, Andrea Modigliani, Ralf M. Palsa}
\affil{European Southern Observatory}
\author{Derek J. McKay}
\affil{Rutherford Appleton Laboratory}
\author{Michael Kiesgen}
\affil{Michael Bailey Assoc.}
\author{Cyrus Sabet}
\affil{Tekom GmbH}

%-----------------------------------------------------------------------
%			 Contact Information
%-----------------------------------------------------------------------
% This information will not appear in the paper but will be used by
% the editors in case you need to be contacted concerning your
% submission.  Enter your name as the contact along with your email
% address.
% 
% EXAMPLE:  \contact{Dennis Crabtree}
%           \email{crabtree@cfht.hawaii.edu}
%

\contact{Yves Jung}
\email{yjung@eso.org}

%-----------------------------------------------------------------------
%		      Author Index Specification
%-----------------------------------------------------------------------
% Specify how each author name should appear in the author index.  The 
% \paindex{ } should be used to indicate the primary author, and the
% \aindex for all other co-authors.  You MUST use the following
% syntax: 
%
% SYNTAX:  \aindex{Lastname, F. M.}
% 
% where F is the first initial and M is the second initial (if
% used).  This guarantees that authors that appear in multiple papers
% will appear only once in the author index.  
%
% EXAMPLE: \paindex{Crabtree, D.}
%          \aindex{Manset, N.}        
%          \aindex{Veillet, C.}        
%
% NOTE: this information is also used to build the author list that
% appears in the table of contents.  Authors will be listed in the order
% of the \paindex and \aindex commmands.
%

\paindex{Jung, Y.}
\aindex{Ballester, P.}  
\aindex{Banse, K.} 
\aindex{Hummel, W.} 
\aindex{Izzo, C.} 
\aindex{McKay, D. J.} 
\aindex{Kiesgen, M.} 
\aindex{Lundin, L. K.} 
\aindex{Modigliani, A.} 
\aindex{Palsa, R. M.} 
\aindex{Sabet, C.} 

%-----------------------------------------------------------------------
%		      Author list for page header	
%-----------------------------------------------------------------------
% Please supply a list of author last names for the page header. in
% one of these formats:
%
% EXAMPLES:
% \authormark{Lastname}
% \authormark{Lastname1 \& Lastname2}
% \authormark{Lastname1, Lastname2, ... \& LastnameN}
% \authormark{Lastname et al.}
%
% Use the "et al." form in the case of seven or more authors, or if
% the preferred form is too long to fit in the header.

\authormark{Jung et al.}

%-----------------------------------------------------------------------
%			Subject Index keywords
%-----------------------------------------------------------------------
% Enter a comma separated list of up to 6 keywords describing your
% paper.  These will NOT be printed as part of your paper; however,
% they will be used to generate the subject index for the proceedings.
% There is no standard list; however, you can consult the indices
% for past proceedings (http://adass.org/adass/proceedings/).
%
% EXAMPLE:  \keywords{visualization, astronomy: radio, parallel
%                     computing, AIPS++, Galactic Center}
%
% In this example, the author noticed that "radio astronomy" appeared
% in the ADASS VII Index as "astronomy" being the major keyword and
% "radio" as the minor keyword.  The colon is used to introduce another
% level into the index.

\keywords{telescopes: VLT, pipelines: data reduction, quality
control, C language, software: data analysis, image: processing}

%-----------------------------------------------------------------------
%			       Abstract
%-----------------------------------------------------------------------
% Type abstract in the space below.  Consult the User Guide and Latex
% Information file for a list of supported macros (e.g. for typesetting 
% special symbols). Do not leave a blank line between \begin{abstract} 
% and the start of your text.

\begin{abstract}          % Leave intact
% Place the text of your abstract here - NO BLANK LINES
Since the beginning of the VLT operations in 1998,
substantial effort has been put in the development of automatic
data reduction tools for the VLT instruments.
A VLT instrument pipeline is a complex system that has to be
able to identify and classify each produced FITS file,
optionally retrieve calibration files from a database, use an
image processing software to reduce the data, compute and
log quality control parameters, produce FITS images or tables
with the correct headers, optionally display them in the
control room and send them to the archive.
Each instrument has its own dedicated pipeline, based on a
common infrastructure and installed with the VLT Data Flow
System (DFS).
With the increase in the number and the complexity of supported
instruments and in the rate of produced data, these pipelines
are becoming vital for both the VLT operations and the users,
and request more and more resources for development and
maintenance.
This paper describes the different pipeline tasks with some
real examples. It also explains how the development process
has been improved to both decrease its cost and increase the
pipelines quality using the lessons learned from the first
instruments pipelines development.
\end{abstract}

%-----------------------------------------------------------------------
%			      Main Body
%-----------------------------------------------------------------------
% Place the text for the main body of the paper here.  You should use
% the \section command to label the various sections; use of
% \subsection is optional.  Significant words in section titles should
% be capitalized.  Sections and subsections will be numbered
% automatically. 
%
% EXAMPLE:  \section{Introduction}
%           ...
%           \subsection{Our View of the World}
%           ...
%           \section{A New Approach}
%
% It is recommended that you look at the sample papers, sample1.tex
% and sample2.tex, for examples for formatting references, footnotes,
% figures, equations, html links, lists, and other special features.  

\section{Introduction}

The VLT instruments pipelines development, integration and maintenance
are tasks under the responsibility of the Data Flow System (DFS) group
at ESO. 

Seven operational instrument pipelines (ISAAC, FORS1, FORS2, NACO, UVES, 
FLAMES, VINCI) are currently under maintenance, and seven others are
being developed and are due over the next two years (CRIRES, VISIR,
SINFONI, VIMOS, GIRAFFE, AMBER, MIDI). On top of that, the second generation
instruments pipelines will have to be developed from 2005 on at a rate
of one pipeline every two years.

\section{Pipelines Infrastructure}

The different pipeline tasks are achieved by a number of different
software components that can be classified in two distinct categories:
those that are common to all instruments and those which are specific to
each instrument pipeline package.

\subsection{The Instrument Independent Components}

The instrument independent components are mainly responsible for the data flow
handling.

\begin{enumerate}
\item The Data Organizer (DO) is responsible for the data flow part. It 
recognizes the incoming FITS files by cross-checking some header keywords with 
{\tt rules} defined in the configuration files delivered with the pipeline
package (see \ref{O10-3:TIPP}). It finds out which calibration files are
needed and retrieves them from the calibration database (also delivered
with the pipeline package). The DO then decides, according to the frames
type, which data reduction recipe should be launched, writes all these
information in a Reduction Block and sends it to the Reduction Block
Scheduler (RBS).

\item The RBS retrieves the created Reduction Blocks, parses them and gives all
the necessary informations to the Data Reduction System (DRS).

\item Some DFS tools are used to send the products to the archive, to write the
computed Quality Control (QC) parameters into the database, etc..
\end{enumerate}

\subsection{The Instrument Pipeline Package}
\label{O10-3:TIPP}

Each pipeline package contains three software components.

\begin{enumerate}
\item The rules needed by the DO to
classify the frames, to know which calibration frames
must be retrieved and which reduction recipe must be used are defined
in the DO configuration files (or {\tt rules}).

\item The calibration frames are contained in the calibration database, which
is the second component of the pipeline package.
Both the calibration database and the rules actually contain instrument
specific information, that cannot be reduced any further.

\item The main software component contained in the Instrument Pipeline Package 
(by far) is of course the DRS. Although many different kinds of data have
to be reduced across the different instruments (see \ref{DR}), the same
low level data structures and data reduction tools are used in many
different pipelines.
Moving these common tasks to a common library would greatly reduce the
size of the different DRS, which is exactly what we are aiming to.
\end{enumerate}

\noindent	%% FO
Of course, the instrument specific components have to be developed for each
different pipeline, and it is clear that our efforts are put on the
reduction of the instrument dependent software size.

\section{The Common Pipeline Library}

The Common Pipeline Library (CPL) (Banse et al. 2004) is a C library that 
contains all these common functionalities. It has been developed during 
the last 2 years, and the first public release will take place in December 
2003. It is one additional Instrument Independent Component that will 
reduce the size of the different DRS delivered with the various pipelines 
packages.

CPL was developed using the C library used for the ISAAC and NACO
pipelines: {\tt eclipse} (Devillard 2001) and the C code developed by
ESO for the VIMOS pipeline integration.
Particular effort has been put on the documentation and on API
clarity, as external consortia will have to base their pipeline
development on this library. From now on, every new instrument pipeline 
will have to be based on CPL.
Besides that, the already existing pipelines written in C (ISAAC, NACO,
VIMOS, etc..) will have to be converted to use CPL.

%The CPL currently contains the very basic operations to handle images,
%tables, FITS headers, matrices or other objects, and it will grow
%according to the new requirements that will come from the new pipelines.
The current content of CPL approximately reflects the needs of the 
pipelines developed so far.

\section{Data Reduction}
\label{DR}

The VLT instruments are each very specific, and their data reduction
requirements vary greatly. The following is a short description 
of the different data reduction {\tt modes}
that can be identified, with their associated instruments.

\subsection{Imaging Mode in Infrared (ISAAC, NACO, VISIR)}

The high background in infrared data must be carefully estimated to
retrieve the science information. In imaging mode, the observations are 
done in {\tt jitter} mode, with small offsets around a central position 
for each exposure, to allow to estimate the sky background variations directly
by filtering the images, and separate astronomical from sky signal. 
Apart from this difficult sky estimation, the frames are recombined with
some cross correlation techniques to precisely determine the offsets
between the images. 

\subsection{Long Slit Spectroscopy in Infrared (ISAAC)}

Like for the imaging mode, the high background is removed using
special observation techniques. In long slit spectroscopy, shifts
are applied along the slit ({\tt nodding}) or a tip/tilt is applied to
the secondary mirror ({\tt chopping}). 
The pipeline must classify and recombine the frames, and then apply 
various calibration corrections like, e.g. wavelength calibration, 
distortion estimation and correction, flat fielding, etc.. The brightest
spectrum is then automatically detected and extracted.

\subsection{Echelle Spectrograph (UVES, CRIRES)}

The UVES science data contain tilted spectra with the different orders
in the same image. To extract all these tilted spectra, a precise
spectral format definition (spectra position and wavelength calibration) 
is needed for all of them. 
This format is determined by different calibration recipes using lamp images 
and physical model solutions.

\subsection{Fiber Mode (FLAMES/UVES)}

In FLAMES, a series of fibers illuminate the slit. They all generate a
spectrum similar to the common UVES observation. These are all stored
together in one common image.
This requires an even more precise spectral format definition to extract each
fiber than for UVES. The reduction for each fiber is then the same as the 
standard one applied for UVES.

\subsection{Multi Object Spectroscopy (VIMOS, FORS1, FORS2)}

The data produced by Multi Object Spectrograph (MOS) contain a huge
number of spectra (up to 800 in VIMOS science images). They all must
be identified and wavelength calibrated. The source spectra are then
individually sky subtracted, flat fielded and integrated.

\subsection{VLTI (MIDI, VINCI, AMBER)}

In the case of VLTI instruments, the data compression rate is very high. 
In the case of MIDI, a 2 gigabytes data set is used to obtain a single 
measurement of the fringes visibility. This means that around 10 measurements 
can be obtained from a night of observations.
The production of a reliable error estimation on the fringes visibility
measurements is a very important task of the pipeline.

\subsection{Integral Field Unit Mode (VIMOS, GIRAFFE, SINFONI)}

The Integral Field Unit (IFU) mode is using fiber bundles or an image
slicer to observe different parts of extended objects. The different fibers'
observations are contained in the same science data, and a very precise
calibration (like for the Fiber mode) is needed to extract the correct
signals.

\section{Quality Control}

The different recipes of the different instrument pipelines produce some
quality control (QC) parameters (Hanuschik et al. 2003) that are automatically
written in a central log file and a common database. 
The health of the instruments is then automatically monitored with e.g. the 
zero  point values, the dark current, the strehl ratio, etc..

%-----------------------------------------------------------------------
%			      References
%-----------------------------------------------------------------------
% List your references below within the reference environment
% (i.e. between the \begin{references} and \end{references} tags).
% Each new reference should begin with a \reference command which sets
% up the proper indentation.  Observe the following order when listing
% bibliographical information for each reference:  author name(s),
% publication year, journal name, volume, and page number for
% articles.  Note that many journal names are available as macros; see
% the User Guide listing "macro-ized" journals.   
%
% EXAMPLE:  \reference Hagiwara, K., \& Zeppenfeld, D.\  1986, 
%                Nucl.Phys., 274, 1
%           \reference H\'enon, M.\  1961, Ann.d'Ap., 24, 369
%           \reference King, I.\ R.\  1966, \aj, 71, 276
%           \reference King, I.\ R.\  1975, in Dynamics of Stellar 
%                Systems, ed.\ A.\ Hayli (Dordrecht: Reidel), 99
%           \reference Tody, D.\  1998, \adassvii, 146
%           \reference Zacharias, N.\ \& Zacharias, M.\ 2003,
%                \adassxii, \paperref{P7.6}
% 
% Note the following tricks used in the example above:
%
%   o  \& is used to format an ampersand symbol (&).
%   o  \'e puts an accent agu over the letter e.  See the User Guide
%      and the sample files for details on formatting special
%      characters.  
%   o  "\ " after a period prevents LaTeX from interpreting the period 
%      as an end of a sentence.
%   o  \aj is a macro that expands to "Astron. J."  See the User Guide
%      for a full list of journal macros
%   o  \adassvii is a macro that expands to the full title, editor,
%      and publishing information for the ADASS VII conference
%      proceedings.  Such macros are defined for ADASS conferences I
%      through XI.
%   o  When referencing a paper in the current volume, use the
%      \adassxii and \paperref macros.  The argument to \paperref is
%      the paper ID code for the paper you are referencing.  See the 
%      note in the "Paper ID Code" section above for details on how to 
%      determine the paper ID code for the paper you reference.  
%
\begin{references}
\reference Banse K. et al.\ 2004, \adassxiii, \paperref{P4-6}
\reference Devillard N.\ 2001, \adassx, 525
\reference Hanuschik R., Hummel W., Sartoretti P., Silva D.\  2003,
Quality Control of the ESO-VLT instruments, Observatory Operations to
Optimize Scientific Return III. Edited by Quinn, Peter J. Proceedings of
the SPIE, Volume 4844, pp. 139-148 (2003)
\end{references}

% Do not place any material after the references section

\end{document}  % Leave intact