%  Corrections FO ==> Mail to ldacosta@eso.org on 07-Feb-2004
%APN3_PROCEEDINGS_FORM%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% TEMPLATE.TEX -- APN3 (2003) ASP Conference Proceedings template.
%
% Derived from ADASS VIII (98) ASP Conference Proceedings template
% Updated by N. Manset for ADASS IX (99), F. Primini for ADASS 2000,
% D.Bohlender for ADASS 2001, and H. Payne for ADASS XII and LaTeX2e.
%
% Use this template to create your proceedings paper in LaTeX format
% by following the instructions given below.  Much of the input will
% be enclosed by braces (i.e., { }).  The percent sign, "%", denotes
% the start of a comment; text after it will be ignored by LaTeX.  
% You might also notice in some of the examples below the use of "\ "
% after a period; this prevents LaTeX from interpreting the period as
% the end of a sentence and putting extra space after it.  
% 
% You should check your paper by processing it with LaTeX.  For
% details about how to run LaTeX as well as how to print out the User
% Guide, consult the README file.  You should also consult the sample
% LaTeX papers, sample1.tex and sample2.tex, for examples of including
% figures, html links, special symbols, and other advanced features.
%
% If you do not have access to the LaTeX software or a laser printer
% at your site, you can still prepare your paper following the
% instructions in the User Guide.  In such cases, the editors will
% process the file and make any necessary editorial adjustments.
% 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\def\void#1{{}} 
%\def\bi{\begin{itemize}}
%\def\ei{\end{itemize}}
%\def\be{\begin{enumerate}}
%\def\ee{\end{enumerate}}
\def\OoneIetal{{\it et al.\/}\ }
%\def\ie{{\it i.e.\/}\rm,\ }
\def\OoneIeg{{\it e.g.\ }}
%\def\lsim{~\rlap{$<$}{\lower 1.0ex\hbox{$\sim$}}}
%\def\gsim{~\rlap{$>$}{\lower 1.0ex\hbox{$\sim$}}}
%
%RECOMMENDED%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% 
\documentclass[11pt,twoside]{article}  % Leave intact
\usepackage{adassconf}

% If you have the old LaTeX 2.09, and not the current LaTeX2e, comment
% out the \documentclass and \usepackage lines above and uncomment
% the following:

%\documentstyle[11pt,twoside,adassconf]{article}

\begin{document}   % Leave intact

%-----------------------------------------------------------------------
%			    Paper ID Code
%-----------------------------------------------------------------------
% Enter the proper paper identification code.  The ID code for your
% paper is the session number associated with your presentation as
% published in the official conference proceedings.  You can           
% find this number locating your abstract in the printed proceedings
% that you received at the meeting or on-line at the conference web
% site; the ID code is the letter/number sequence proceeding the title 
% of your presentation. 
%
% This will not appear in your paper; however, it allows different
% papers in the proceedings to cross-reference each other.  Note that
% you should only have one \paperID, and it should not include a
% trailing period.
%
% EXAMPLE: \paperID{O4-1}
% EXAMPLE: \paperID{P7-7}
%

\paperID{O1-1}
%%%% ID=O1-1

%-----------------------------------------------------------------------
%		            Paper Title 
%-----------------------------------------------------------------------
% Enter the title of the paper.
%
% EXAMPLE: \title{A Breakthrough in Astronomical Software Development}
% 
% If your title is so long as to fill the page header when you print it,
% then please supply a short form as a \titlemark.
%
% EXAMPLE: 
%  \title{Rapid Development for Distributed Computing, with Implications
%         for the Virtual Observatory}
%  \titlemark{Rapid Development for Distributed Computing}
%

\title{The ESO Imaging Survey Project: Building a Survey Software System}
\titlemark{ Software System for ESO Imaging Survey Project } %% FO

%-----------------------------------------------------------------------
%		          Authors of Paper
%-----------------------------------------------------------------------
% Enter the authors followed by their affiliations.  The \author and
% \affil commands may appear multiple times as necessary (see example
% below).  List each author by giving the first name or initials first
% followed by the last name.  Authors with the same affiliations
% should grouped together. 
%
% EXAMPLE: \author{Raymond Plante, Doug Roberts, 
%                  R.\ M.\ Crutcher\altaffilmark{1}}
%          \affil{National Center for Supercomputing Applications, 
%                 University of Illinois Urbana-Champaign, Urbana, IL
%                 61801}
%          \author{Tom Troland}
%          \affil{University of Kentucky}
%
%          \altaffiltext{1}{Astronomy Department, UIUC}
%
% In this example, the first three authors, "Plante", "Roberts", and
% "Crutcher" are affiliated with "NCSA".  "Crutcher" has an alternate 
% affiliation with the "Astronomy Department".  The fourth author,
% "Troland", is affiliated with "University of Kentucky"

\author{Luiz da Costa, Charles Rite and Remco G. Slijkhuis}
\affil{European Southern Observatory, Karl-Schwarzschild-Str 2,
  Garching bei M\"unchen, D-85748, Germany }

%-----------------------------------------------------------------------
%			 Contact Information
%-----------------------------------------------------------------------
% This information will not appear in the paper but will be used by
% the editors in case you need to be contacted concerning your
% submission.  Enter your name as the contact along with your email
% address.
% 
% EXAMPLE:  \contact{Dennis Crabtree}
%           \email{crabtree@cfht.hawaii.edu}
%

\contact{L. da Costa }
\email{ldacosta@eso.org }

%-----------------------------------------------------------------------
%		      Author Index Specification
%-----------------------------------------------------------------------
% Specify how each author name should appear in the author index.  The 
% \paindex{ } should be used to indicate the primary author, and the
% \aindex for all other co-authors.  You MUST use the following
% syntax: 
%
% SYNTAX:  \aindex{Lastname, F. M.}
% 
% where F is the first initial and M is the second initial (if
% used).  This guarantees that authors that appear in multiple papers
% will appear only once in the author index.  
%
% EXAMPLE: \paindex{Crabtree, D.}
%          \aindex{Manset, N.}        
%          \aindex{Veillet, C.}        
%
% NOTE: this information is also used to build the author list that
% appears in the table of contents.  Authors will be listed in the order
% of the \paindex and \aindex commmands.
%

\paindex{da Costa, L.}
\aindex{Rite, C.}
\aindex{Slijkhuis, R. G.}

%-----------------------------------------------------------------------
%		      Author list for page header	
%-----------------------------------------------------------------------
% Please supply a list of author last names for the page header. in
% one of these formats:
%
% EXAMPLES:
% \authormark{Lastname}
% \authormark{Lastname1 \& Lastname2}
% \authormark{Lastname1, Lastname2, ... \& LastnameN}
% \authormark{Lastname et al.}
%
% Use the "et al." form in the case of seven or more authors, or if
% the preferred form is too long to fit in the header.

\authormark{da Costa et al.}

%-----------------------------------------------------------------------
%			Subject Index keywords
%-----------------------------------------------------------------------
% Enter a comma separated list of up to 6 keywords describing your
% paper.  These will NOT be printed as part of your paper; however,
% they will be used to generate the subject index for the proceedings.
% There is no standard list; however, you can consult the indices
% for past proceedings (http://adass.org/adass/proceedings/).
%
% EXAMPLE:  \keywords{visualization, astronomy: radio, parallel
%                     computing, AIPS++, Galactic Center}
%
% In this example, the author noticed that "radio astronomy" appeared
% in the ADASS VII Index as "astronomy" being the major keyword and
% "radio" as the minor keyword.  The colon is used to introduce another
% level into the index.

\keywords{surveys, EIS, software}

%-----------------------------------------------------------------------
%			       Abstract
%-----------------------------------------------------------------------
% Type abstract in the space below.  Consult the User Guide and Latex
% Information file for a list of supported macros (e.g. for typesetting 
% special symbols). Do not leave a blank line between \begin{abstract} 
% and the start of your text.

\begin{abstract}          % Leave intact
% Place the text of your abstract here - NO BLANK LINES

To cope with the rapidly increasing volume and complexity of data
being generated by ESO's public optical/infrared imaging surveys a
high-performance, end-to-end survey software system has been
developed. Its main aim is to provide a robust framework to
efficiently transform raw data into science-grade survey products. The
system consists of several tasks wrapped together into an integrated
framework. These tasks include: the un-supervised reduction of
optical/infrared images generated by different imagers, the
astrometric and photometric calibration of the data, the creation of
image stacks and mosaics, the preparation of catalogs and the
selection of different targets of potential interest for spectroscopic
follow-up. Since the data are meant to be public, the system also
provides an extensive description of the products and the required
information for users to assess their quality. The system has been
designed to enable a small group to monitor the un-supervised
reduction and analysis of data from multiple surveys in their entirety
-- from survey definition all the way through to the release of
comprehensively documented survey products (stacked images, mosaics
and catalogs) -- all done by one operator from a single desktop. While
originally designed for handling survey data, the system can also be
used as a general front-end to ESO's raw data archive, and as such
serve as a site-specific interface to the general VO
infra-structure. In this contribution the system is briefly described.

\end{abstract}

%-----------------------------------------------------------------------
%			      Main Body
%-----------------------------------------------------------------------
% Place the text for the main body of the paper here.  You should use
% the \section command to label the various sections; use of
% \subsection is optional.  Significant words in section titles should
% be capitalized.  Sections and subsections will be numbered
% automatically. 
%
% EXAMPLE:  \section{Introduction}
%           ...
%           \subsection{Our View of the World}
%           ...
%           \section{A New Approach}
%
% It is recommended that you look at the sample papers, sample1.tex
% and sample2.tex, for examples for formatting references, footnotes,
% figures, equations, html links, lists, and other special features.  

\section{Introduction}

Recently, there has been a renewed interest in carrying out large
optical/infrared imaging surveys. Several factors have contributed to
instigate this interest: the commissioning of several
large-aperture telescopes, and the demand that this has created for
the preparation of suitable data sets matching their spectroscopic
capabilities, the coming-of-age of modern large optical/infrared
arrays leading to the construction of cameras with fields of view on
degree scales, and the assignment of dedicated imaging
telescopes. Combined, these new developments have made
multi-wavelength, digital surveys covering large areas of the sky
possible. Moreover, they represent a marked improvement in terms of
speed, depth and quality over the older generation of photographic
plate surveys.  In addition to these traditional surveys, the
implementation of ever-growing digital raw data archives to store data
from proprietary programs by some of the major observatories also
offers new science opportunities. Together,  these developments have created a
glut of data and currently the main challenge is how to cope with
the large increase in data volume available and allow the scientific
exploitation of these federated archives.


Fortunately, progress in IT has also been unprecedented in the past
decade and the rapid increase in computer power and storage
capabilities on the one hand, and the development of new technologies
on the other, offer the means to meet the challenge posed by the large
data volumes involved. However, as discussed below new software
systems must be developed to successfully handle the new generation
of surveys, which can take the form of: 1) small area, deep
multi-wavelength surveys, involving a variety of space- and
ground-based instruments from different observatories; 2) wide-angle,
moderately deep, legacy-type surveys covering large swathes of the sky;
3) virtual surveys relying on archival data. These different types of
survey, combined with existing institutional infra-structure and
resources, and target audience, set the requirements that must be taken
into account in designing a suitable survey system. 

At this point it is important to underscore the difference between
{\it``pipelines''} put together to reduce data from {\it ``survey
systems''} which are intended to provide a comprehensive environment
to define, control, reduce, analyze and monitor the quality
of data, produce a range of survey products and make them publicly
available.  While the former may suffice for specific science groups
with clear objectives and a finite amount of data, the latter is
required to support long-term public surveys producing readily
accessible information and self-descriptive, quality assessed,
homogeneous survey products ready for scientific exploitation.

\void{
in general have different timescales and different end users, leading
to different requirements on the design of the system to cope with the
complexity and/or large volumes of data generated by these surveys.
 The requirements on the system is set both by
the nature of the surveys being considered, the target end-user and
the environment where it will be operated. At this point it is
important to make a clear distinction between ``pipelines'' put
together to reduce data from survey systems which are intended to
provide a comprehensive environment to define and monitor surveys,
reduce, analyze and monitor the quality of data, produce a range of
survey products, making them publicly available.
}

In this contribution, the work being carried out by the ESO Imaging
Survey (EIS) project (Renzini \& da Costa 1997) to build such a survey
system is discussed. In addition to carrying out a large number of
public surveys, this project has been involved for the past three
years in the development of software required to reduce and
administrate imaging data from imaging surveys, involving different
imagers. In section~\ref{O1-1:integrated} the requirements for such a
system are briefly reviewed, while in section~\ref{O1-1:eis} some of the
main features of the EIS survey system are presented. To illustrate
the advantages of an integrated system, in section~\ref{O1-1:operation} the
operation of the system for survey work is briefly described. Finally,
in section~\ref{O1-1:summary} the main achievements of this development are
summarized.


\section {Building an Integrated System}
\label{O1-1:integrated}

The ESO public survey effort started in July 1997 prior to the
commissioning of VLT, and its first phase was completed at the end of
1998 with the full release of the optical and infrared data
accumulated for the EIS-WIDE and EIS-DEEP surveys conducted using the
NTT at La Silla. Besides astrometrically and photometrically
calibrated pixel maps, the release involved the delivery of a host of
survey products which included image stacks and mosaics, single and
multi-passband catalogs for stacks and mosaics, and lists of candidate
clusters of galaxies, quasars, white dwarfs and other color-selected
targets, meeting all the requirements and the main deadline set by the
Public Survey Working Group, with the delivery taking place prior to
the start of commissioning and operations of the first VLT unit in
December 1998.

These original reductions were carried out using adaptations of
pre-existing software (\OoneIeg IRAF, Eclipse, SExtractor, Drizzle, LDAC),
some by the original authors who participated in kick-starting EIS. To
facilitate the data reduction these various modules were then
interconnected using simple scripting languages. Most of the
reductions were carried out by people with considerable experience in
data processing. While successfully meeting the goals set for this
experimental project, it was carried out on a best-effort basis, the
experience accumulated during its execution and aftermath
unequivocally demonstrated that unless a proper system was available
this could only be a one-off effort, unsustainable over long stretches
of time. This became abundantly clear with the start of operations of
the wide-field imager (WFI) in 1999 at La Silla and the increased
complexity of the survey strategies adopted, usually involving more
than one instrument.

In summary, the major legacy of the experience accumulated by EIS in
the first three-year long phase of the project was that it clearly
revealed the scope of the enterprise and the broad range of
requirements for successfully conducting extensive and truly {\it
public} imaging surveys, which requires proper handling not only of
data but also of information, to facilitate the visualization and
monitoring of the surveys by interested users. To address these needs,
since June 2000 a major effort has been underway to develop an
end-to-end, fully integrated survey system.  The main objective has
been to develop a system capable of:

\begin{itemize}

\item sustaining un-supervised, 24/7 operation cycle over
  long stretches of time required to enable the reduction of a nights
  worth of data from VST and VISTA in one day

\item providing a user-friendly and, as much as possible, menu-driven
system so as to minimize the need for documentation and ease the
training of new users and operators

\item  reducing data from different instruments (single-, multi-chip)
 and wavelength domains

\item producing uniform, well-documented and VO-compatible primary and
  derived products

\item providing a framework to monitor the progress of observations
and reductions

\item providing mechanisms for the automatic update of WEB pages to
serve as interfaces to the end-users of survey data

\end{itemize}



\begin{figure}
%\epsscale{.80}
%\plotone{dacostafig2.ps}
\plotone{O1-1_fig1.ps}
\caption[]{Schematic view of the {\bf integrated} EIS survey software system
showing some of the system's key components.}
\label{O1-1:fig:scheme}
\end{figure}

\noindent Other important requirements that the system had to  fulfill were:


\begin{itemize}

\item high-throughput

\item configurable algorithms

\item support ``versioning'' of software and data products, 
 and provide history of the  products, software and system

\item flexible tools to handle and visualize system products

\item self-descriptive products with quality control assessment

\item highly automated, end-to-end, and integrated environment

\end{itemize}


\noindent %% FO
The development of the system was split into the following parts:


\begin{enumerate}

\item  a high-performance specially-designed C$^{++}$-based image
 processing system, consisting of over 100,000 lines of code, to
 reduce optical/infrared images from single- and multi-chip
 instruments, referred to hereafter as the EIS/MVM
 (multi-visualisation-model) system (\OoneIeg Vandame 2002). This system
 also includes advanced techniques to efficiently register (using
 wavelet transform techniques) and de-fringe images, correct for
 scattered light and chip-to-chip gain variations, and remove cosmic
 rays and satellite tracks (using Hough transform), leading to
 cosmetically enhanced final images, which of course depend on
 passband and observing strategy.

\item  a Python wrapper consisting of over 250,000 lines of code,
 responsible for the administration of the entire system and
 interfacing it to the EIS internal database (Sybase), consisting of
 over 100 tables, different visualization tools, analysis plug-ins,
 the WEB browser, action request system (ARS) and the ESO data flow
 infrastructure.

\item graphic user interfaces, constructed using Tcl/Tk, in order to
 allow easy access by the user to different tasks.

\item extensive use of  XML (extensible  mark-up language) and SVG
 (scalable vector graphics) technologies in
 the preparation of configuration, logs, and WEB pages, for both
 internal as well as external use.

\end{enumerate}


\noindent %% FO
Figure~\ref{O1-1:fig:scheme} gives a schematic view of the various
interfaces that had to be developed in the construction of the
integrated system being described.

\void{
From the start the system was designed around To provide the required
oversight of the system, any action that changes the status of the
internal database or require the intervention of the system operator,
is followed by a pipeline alert message sent to the ARSystem being
employed which also notifies the operator and system administrator.  }


\section {The EIS survey system}
\label{O1-1:eis}


\noindent %% FO
The end-to-end EIS survey system consists of distinct modules, each
representing a {\it system} process as well as a possible entry point
into the pipeline framework. The system modules fall into the
following distinct categories

\bigskip
\noindent {\bf Front-end}

\void{
 The first group,
representing the front-end of the system, consists of the
following modules:}

\begin{enumerate}

\item Scanner: scans the ESO Science Archive to identify exposures
belonging to one of the survey programs. When these are identified it
triggers the update of several internal tables, computes some basic
properties of these exposures (\OoneIeg moon distance, DIMM seeing) and the
nights (\OoneIeg twilight, lunar phase) when they were observed,
reconstructs the observing blocks used in the observations, creates
(updates) night, run and survey summary logs, in the form of XML
files, which are published on the WEB prior to data request, thus
providing a {\it real-time} status report to external users.

\item Data request: requests raw data from  the ESO science
archive. The process retrieves data from the storage medium, which are
then sent across the network to target directories specified by the
user or set automatically by the system resource manager by FTP, and
uncompressed. After all the data is transferred, the process sends an
alert to the ARS and, if so configured, launches the image reduction
pipeline automatically.

\item Image Reduction: checks the integrity of the FITS files, the
existence and content of mandatory keywords used by the reduction
package; groups the data by night, passband, pointing and time
sequence; create reduction blocks according to well-defined and
configurable rules.  It then interfaces with the C-based image
processing software via XML configuration files. At the end, it
computes several attributes for the final image which are fully
described in an XML file representing the associated {\it product
log}. Different style-sheets are used depending from where these logs
are retrieved, with those internal to the system reporting more
detailed administrative information of no relevance to external users.

\item Data Calibration \& Photometric Pipeline; separates exposures
taken of selected fields containing photometric standard stars,
extracts catalogs and matches measured stars with those available in
tables of the internal database to identify standards and recover
information about them as given in the literature.

\end{enumerate}

\noindent %% FO
After reduction, the final products are inspected and graded using a
specialized tool to control the quality of the products, the data are
transferred to an image repository and the raw data are deleted from
disk.

\bigskip
\noindent {\bf Back-end}


The second group of modules forms the back-end of the system from
where the results of different observations are combined into final
image stacks or mosaics, and from where catalogs are extracted,
targets selected and survey products released. The main modules are:


\begin{enumerate}

\item Image products: creates stack/mosaic blocks according to
well-defined rules and validation procedures from which stack/mosaic
images. The process also allows, in batch mode, to create science
grade catalogs; compute final image attributes and assign grade to
final products either automatically or by visual inspection


\item Catalog production: extracts single passband catalogs using
either SExtractor or an adapted version of DAOPHOT and PSFEX (Bertin
2003), the latter on an experimental basis; prepares science grade
single passband, mosaic and color catalogs, the latter using either a
reference image (specified by the user) or by association of single
passband catalogs.


\item Analysis: where analysis code will be plugged-in. This  process
will include matched-filter for cluster finding, mass reconstruction
algorithms from PSF distortions, photometric redshifts and target
classification using color criteria and/or template fitting.

\item Data Release: moves survey products back to ESO Science Archive;
updates WEB pages including the release index table and request form;
links survey products to their respective logs; creates entries in
image gallery and sends an alert to the relevant people

\end{enumerate}

\noindent %% FO
Again, final results can be examined using a quality control tools
similar to that mentioned above, which also allows the examination of
the history of a product and the difference between versions of the
same product.

\bigskip
\noindent {\bf Administration}

\begin{enumerate}

\item Administration: access to system, survey,  database.  WEB and
action request system tools and interfaces. From this location one has
access to data and software CVS repositories, on-line documentation,
among others.

\item Observations: survey definition (strategy, regions, fields,
filter, integration time), creation of observing blocks and finding
charts; extraction of subsets of reference catalogs and other all-sky
catalogs; computation of region properties (number of bright stars, HI
column density, galactic absorption, galactic model predictions of
star counts computed using the response function of the filters used
in the observations).

\end{enumerate}


\noindent %% FO
The system supports different modes of operation: 1) interactive: used
primarily for testing and fine-tuning of configuration parameters; 2)
automatic: allows a process to be executed end-to-end without user
intervention; 3) batch: enables a pre-determined sequence of processes
to be executed in sequence. Batches are available both in the front-
and back-end of the system, providing enormous flexibility in its
operation - some batches are ideal for un-supervised survey
operations, while others are more appropriate for end-user applications.

It is important to note that the design of the system is still in
progress and continues to evolve as more functionalities have been
included as a result of the experience gained in the operation of the
system and from suggestions made  by test-users.

To each module there corresponds a graphic user interface (GUI) which
allows the user to interact with the system in the interactive mode,
or to monitor the progress of the processes in the automatic or batch
mode. The individual panels can be called from a single widget,
displayed at log-in. The widget not only provides the access to the
various panels but also reports the version of the system being used,
based on the version control system CVS, and the date when the system
was last updated.

In batch mode, at the end of each process the corresponding panel is
automatically iconized, while the next in the pre-defined batch
sequence is launched, thus preventing overcrowding of the
workspace. When a new panel is launched, temporary disk directories
are created to store all the files generated by the process. These are
automatically deleted when the panel is closed.

Each module has an interface to the system's supporting database
(hereafter DB) and to a data access layer (DAL) which is fed by a
search engine. The latter supports generic queries to locate the data
suitable to the specific process being considered. The supported
queries are combinations of survey, instrument, passband and sky
region (pre-defined in the case of surveys).  The results of a given
query provide a list of entries (\OoneIeg runs, images, catalogs) followed
by information describing them, from where the user may select any
number of entries.  For entries with more than one version the user
may select either the most recent or a default version which can also
be set by the user a priori. From the DAL it will soon also be
possible to apply other more generic constraints (\OoneIeg data ownership,
dates, quality of the data) that will enable further culling of the data
accepted by a given process. These constraints will allow the system
to be used in a more generic way for different applications

An example of the layout of a panel is shown in Figure~\ref{O1-1:fig:ip}
which illustrates the image reduction panel. The layout is typical of
all panels with small variations depending on the specific
process. The design is preliminary and as mentioned earlier is still
evolving. The top part of the panel is split into three sections. The
first lists all the sub-processes that can be executed. It also allows
the user to set the mode of operation (batch/automatic/interactive)
and, in some cases, the type of process to be executed.  In the banner
of the panel one finds the name of the process, the execution mode,
the revision of the code and the user.

\begin{figure}
%\epsscale{.80}
%\plotone{dacostafig1.ps}
\plotone{O1-1_fig2.ps}
\caption[]{Example of the GUI used throughout the EIS system. The one
shown is that of the image pipeline panel.}
\label{O1-1:fig:ip}
\end{figure}


From the second section one can access the configuration file and the
search-engine/DAL combination, described above. The configuration is
presented as an HTML form consisting of a combination of field boxes,
radial and pull-down buttons depending on the nature of the
information to be provided. The configuration file displayed depends
on the mode of operation selected. In interactive/automatic mode only
parameters referring to the specific process being called are
shown. In batch mode, the configuration is the collection of all
configurations of the processes being executed in sequence. These are
shown in distinct, superposed HTML forms. In the banner of the
configuration browser, information about the configuration file is
displayed including user name, creation date and the type (\OoneIeg last
used, user's default, system's default).  When a process is completed
and saved, the configuration file used is ingested into the database
and can be accessed from the process log, thus providing a link
between process, configuration, input data and product, which is at
the core of the versioning mechanism of the system.

Finally, on each panel a set of keys are available under the
administration section. With the exception of that labeled PANEL
TOOLS, all others are common to all panels and provide short-cuts to a
variety of administration tools for the pipeline, survey, WEB and
database.  Under PANEL TOOLS there is a large variety of tools
specific to each panel. The last key, labeled save is used at the end
of the process to ingest the required information into the EIS DB and
move final products to appropriate directories.

The panels also include a TTY display, where some of the more relevant
information about the process being executed is reported. In addition,
on top of the TTY one has: a status button, reporting the name of the
sub-process being executed; a progress bar, reporting the progress of
the sub-process; a data rate meter reporting the mean or the
instantaneous data rate, whenever possible; and a clock which reports
the elapsed time for each sub-process. In the process log both the
mean data rate and time fraction spent on each sub-process is reported
so as to enable the monitoring of the performance of the system over
time.

The lower part of the panel consists of listboxes where the results of
most sub-processes of the panel are listed. Next to each listbox there
are keys that call tasks that allow the user to examine intermediate
and final products in various ways depending on their nature. These
keys are associated with tasks to display images and catalogs, to show
other listboxes providing more details about each entry, and to
convert XMLs into HTMLs and display process and product logs. In the
example shown in Figure~\ref{O1-1:fig:ip}, there are four listboxes listing
different runs (upper-left) and raw images (upper-right), groups
(lower-left) and reduction blocks (lower-right), a collection of raw
dithered exposures that should be reduced and stacked together, for
the selected run.



\section {Survey Operation Model}
\label{O1-1:operation}

A key requirement in the design of the survey system has been to
minimize the need for human intervention, at the same time providing
all the required information to facilitate the monitoring of the
performance of the system. For surveys the sequence of operations is
approximately as follows:

\begin{enumerate}

\item scan archive for exposures having a  survey program-id, notify
operator via the action request system (ARS) and trigger data request one
night at a time

\item if new exposures are identified, information describing them is
immediately ingested into the EIS database, summaries created and
updated pages are published showing the progress of survey in the
WEB. Optionally, at the end the process triggers the request of the newly
arrived data

\item data for the same instrument and night are sent to all available
machines in the cluster and the image reduction process
triggered. Images are processed on a nightly basis, astrometrically
and photometrically calibrated and the operator notified via ARS to carry
out the quality control. After grading the final products these are
moved to proper repositories, for future reference and inspection, and
the raw data deleted from disk, thus allowing new data to come in

\end{enumerate}

\noindent %% FO
At the end of an observing season the following steps are taken:

\begin{enumerate}

\item define (update) new (ongoing) surveys and export new observing
blocks, as required by the data flow system of ESO, to the telescope team

\item create final stacks/mosaics, extract catalogs and select targets
on a survey basis

\item examine results and assign grade reflecting quality of the
product

\item ingest and move products to repositories

\item release version 0.5 of advanced survey products

\end{enumerate}

\noindent %% FO
Finally, monitor the comments received via the ARSystem from external
users, and if necessary release revised versions of the final products
including XML logs showing the differences in the results and in the
configuration files used in their definition.


While new functionalities are being continuously added to the system,
tests of both the C-based image processing pipeline and the survey
system have been underway for the past two years.

The image processing pipeline has been used to reduce large
amounts of data from most ESO imagers (SOFI, ISAAC, WFI, FORS) and
data from these reductions have been publicly released. Since February
2001, a total of six releases have been made illustrating the data
reduction for four different surveys using different instruments and
strategies, especially in the Chandra Deep South field (\OoneIeg Arnouts
\OoneIetal 2001, Vandame \OoneIetal 2003) and selected stellar fields (\OoneIeg Momany
\OoneIetal 2001). The  system has also had a limited distribution to external
users for tests, and extensive comparisons with reductions done using
different softwares have been made for for multi-chip optical data and
infrared data (\OoneIeg DIMSUM). The system has also been benchmarked in
different platforms (SUN, Compaq Alphas, PCs). Currently, one Linux
box running REDHAT and 2 Alpha running TRUE-64 are dedicated for this
offline test reductions and code development.

In parallel, tests with the survey system are being carried out using
for the moment only single chip instruments (SOFI and ISAAC). A total
of about 39,000 SOFI frames, taken since 1998, and 10,000 ISAAC frames
(which combined total about 100 GB) have been reduced several times,
in order to identify exceptional situations and make the code robust,
as required for un-supervised reduction .  These tests are being
conducted using three dual-CPU ($\sim$ 2 GHz), number-crunching Linux
boxes, which will soon be expanded to six to form the operation
environment of EIS. At the present time, different sessions
(processes) are launched on each individual system and monitored from
a single desktop using VNC (virtual network computing) software. These
sessions are being launched manually, but hopefully soon this will be
done automatically using the system developed by the CONDOR project,
which provides a resource monitoring and management, and scheduling
and job queuing mechanism.  This hardware/software environment
provides an effective reduction data rate of about 0.5 to 4 Mpix/sec
and can be fully operated by a single user. While the system is
essentially in operation, a lot of effort is being made to make the
code uniform, with the same look \& feel, uniform and comprehensive
logs and the required tools to evaluate the quality of the suite of
survey products being produced. Several successful public
demonstrations of the system in operation have been made over the past
year. Progress has been hampered by the limited resources of the
development team and the turn-around of team members.


\section {Summary}
\label{O1-1:summary}

In this contribution some highlights of the EIS survey system have
been reviewed. Even though the system is a work in progress, it
currently supports un-supervised and automated operations, to
efficiently reduce optical/infrared data from single/multi-chip
instruments.  While originally designed for survey operations,
especially public surveys, it is currently being generalized to deal
with archival data and individual end-users. 

A particular important characteristic of the system design has been to
provide a general infrastructure to allow different tools to be
integrated so as to facilitate the administration of the surveys,
primary and advanced survey products and of the system itself. It is
fully integrated into the available data flow infrastructure of ESO
which will make it possible to use the same system as a portal to the
ESO Science archive interfacing it to the Virtual Observatory
infrastructure.  In fact, the back-end of a survey system would
greatly benefit from VO-like tools for the assessment of the quality
of the survey products.

\acknowledgments

It is a pleasure to thank all past (too many to list here) and current
members (P. Lynam, A. Mignano,  V. Strazzullo,  B. Vandame) of the EIS
team as well as those that continue to collaborate with the effort
from their home institutes including S. Arnouts (Marseille),
C. Benoist (Nice), L. Girardi (Trieste), L. F. Olsen (Copenhagen),
S. Zaggia (Trieste).


%-----------------------------------------------------------------------
%			      References
%-----------------------------------------------------------------------
% List your references below within the reference environment
% (i.e. between the \begin{references} and \end{references} tags).
% Each new reference should begin with a \reference command which sets
% up the proper indentation.  Observe the following order when listing
% bibliographical information for each reference:  author name(s),
% publication year, journal name, volume, and page number for
% articles.  Note that many journal names are available as macros; see
% the User Guide listing "macro-ized" journals.   
%
% EXAMPLE:  \reference Hagiwara, K., \& Zeppenfeld, D.\  1986, 
%                Nucl.Phys., 274, 1
%           \reference H\'enon, M.\  1961, Ann.d'Ap., 24, 369
%           \reference King, I.\ R.\  1966, \aj, 71, 276
%           \reference King, I.\ R.\  1975, in Dynamics of Stellar 
%                Systems, ed.\ A.\ Hayli (Dordrecht: Reidel), 99
%           \reference Tody, D.\  1998, \adassvii, 146
%           \reference Zacharias, N.\ \& Zacharias, M.\ 2003,
%                \adassxii, \paperref{P7.6}
% 
% Note the following tricks used in the example above:
%
%   o  \& is used to format an ampersand symbol (&).
%   o  \'e puts an accent agu over the letter e.  See the User Guide
%      and the sample files for details on formatting special
%      characters.  
%   o  "\ " after a period prevents LaTeX from interpreting the period 
%      as an end of a sentence.
%   o  \aj is a macro that expands to "Astron. J."  See the User Guide
%      for a full list of journal macros
%   o  \adassvii is a macro that expands to the full title, editor,
%      and publishing information for the ADASS VII conference
%      proceedings.  Such macros are defined for ADASS conferences I
%      through XI.
%   o  When referencing a paper in the current volume, use the
%      \adassxii and \paperref macros.  The argument to \paperref is
%      the paper ID code for the paper you are referencing.  See the 
%      note in the "Paper ID Code" section above for details on how to 
%      determine the paper ID code for the paper you reference.  
%
\begin{references}

\reference  Arnouts, S., Vandame, B., Benoist, C., \OoneIetal,  2001, A\&A, 379, 740

\reference Bertin, E., 2003, {\it private communication.}

\reference Momany, Y., Vandame, B., Zaggia, S., \OoneIetal ,  2001, A\&A, 379, 436

\reference Renzini, A. \& da Costa, L., 1997, The Messenger,
87, 23

\reference Vandame, B. \ 2002, Astronomical Data Analysis II, eds
(J.L. Stark, F. D. Murtagh) Proceedings of the SPIE, 4847, p. 123

\reference Vandame, B. \OoneIetal  \ 2003, A\&A, {\it submitted}
 (astro-ph/0102300)

\end{references}

% Do not place any material after the references section

\end{document}  % Leave intact
