%APN3_PROCEEDINGS_FORM%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% TEMPLATE.TEX -- APN3 (2003) ASP Conference Proceedings template.
%
% Derived from ADASS VIII (98) ASP Conference Proceedings template
% Updated by N. Manset for ADASS IX (99), F. Primini for ADASS 2000,
% D.Bohlender for ADASS 2001, and H. Payne for ADASS XII and LaTeX2e.
%
% Use this template to create your proceedings paper in LaTeX format
% by following the instructions given below.  Much of the input will
% be enclosed by braces (i.e., { }).  The percent sign, "%", denotes
% the start of a comment; text after it will be ignored by LaTeX.  
% You might also notice in some of the examples below the use of "\ "
% after a period; this prevents LaTeX from interpreting the period as
% the end of a sentence and putting extra space after it.  
% 
% You should check your paper by processing it with LaTeX.  For
% details about how to run LaTeX as well as how to print out the User
% Guide, consult the README file.  You should also consult the sample
% LaTeX papers, sample1.tex and sample2.tex, for examples of including
% figures, html links, special symbols, and other advanced features.
%
% If you do not have access to the LaTeX software or a laser printer
% at your site, you can still prepare your paper following the
% instructions in the User Guide.  In such cases, the editors will
% process the file and make any necessary editorial adjustments.
% 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% 
\documentclass[11pt,twoside]{article}  % Leave intact
\usepackage{adassconf}

% If you have the old LaTeX 2.09, and not the current LaTeX2e, comment
% out the \documentclass and \usepackage lines above and uncomment
% the following:

%\documentstyle[11pt,twoside,adassconf]{article}

\begin{document}   % Leave intact

%-----------------------------------------------------------------------
%			    Paper ID Code
%-----------------------------------------------------------------------
% Enter the proper paper identification code.  The ID code for your
% paper is the session number associated with your presentation as
% published in the official conference proceedings.  You can           
% find this number locating your abstract in the printed proceedings
% that you received at the meeting or on-line at the conference web
% site; the ID code is the letter/number sequence proceeding the title 
% of your presentation. 
%
% This will not appear in your paper; however, it allows different
% papers in the proceedings to cross-reference each other.  Note that
% you should only have one \paperID, and it should not include a
% trailing period.
%
% EXAMPLE: \paperID{O4-1}
% EXAMPLE: \paperID{P7-7}
%

\paperID{P4-25}
%%%% ID=P4-25

%-----------------------------------------------------------------------
%		            Paper Title 
%-----------------------------------------------------------------------
% Enter the title of the paper.
%
% EXAMPLE: \title{A Breakthrough in Astronomical Software Development}
% 
% If your title is so long as to fill the page header when you print it,
% then please supply a short form as a \titlemark.
%
% EXAMPLE: 
%  \title{Rapid Development for Distributed Computing, with Implications
%         for the Virtual Observatory}
%  \titlemark{Rapid Development for Distributed Computing}
%

\title{Extending ORAC-DR for Offline Processing of ESO, INGRID, and
Classic Cam data}
\titlemark{Extending ORAC-DR for Offline Processing}

%-----------------------------------------------------------------------
%		          Authors of Paper
%-----------------------------------------------------------------------
% Enter the authors followed by their affiliations.  The \author and
% \affil commands may appear multiple times as necessary (see example
% below).  List each author by giving the first name or initials first
% followed by the last name.  Authors with the same affiliations
% should grouped together. 
%
% EXAMPLE: \author{Raymond Plante, Doug Roberts, 
%                  R.\ M.\ Crutcher\altaffilmark{1}}
%          \affil{National Center for Supercomputing Applications, 
%                 University of Illinois Urbana-Champaign, Urbana, IL
%                 61801}
%          \author{Tom Troland}
%          \affil{University of Kentucky}
%
%          \altaffiltext{1}{Astronomy Department, UIUC}
%
% In this example, the first three authors, "Plante", "Roberts", and
% "Crutcher" are affiliated with "NCSA".  "Crutcher" has an alternate 
% affiliation with the "Astronomy Department".  The fourth author,
% "Troland", is affiliated with "University of Kentucky"

\author{Malcolm J. Currie}
\affil{Starlink, Rutherford Appleton Laboratory, Chilton, Didcot,
Oxfordshire OX11 0QX, U.K.}

%-----------------------------------------------------------------------
%			 Contact Information
%-----------------------------------------------------------------------
% This information will not appear in the paper but will be used by
% the editors in case you need to be contacted concerning your
% submission.  Enter your name as the contact along with your email
% address.
% 
% EXAMPLE:  \contact{Dennis Crabtree}
%           \email{crabtree@cfht.hawaii.edu}
%

\contact{Malcolm Currie}
\email{mjc@star.rl.ac.uk}

%-----------------------------------------------------------------------
%		      Author Index Specification
%-----------------------------------------------------------------------
% Specify how each author name should appear in the author index.  The 
% \paindex{ } should be used to indicate the primary author, and the
% \aindex for all other co-authors.  You MUST use the following
% syntax: 
%
% SYNTAX:  \aindex{Lastname, F. M.}
% 
% where F is the first initial and M is the second initial (if
% used).  This guarantees that authors that appear in multiple papers
% will appear only once in the author index.  
%
% EXAMPLE: \paindex{Crabtree, D.}
%          \aindex{Manset, N.}        
%          \aindex{Veillet, C.}        
%
% NOTE: this information is also used to build the author list that
% appears in the table of contents.  Authors will be listed in the order
% of the \paindex and \aindex commmands.
%

\paindex{Currie, M. J.}

%-----------------------------------------------------------------------
%		      Author list for page header	
%-----------------------------------------------------------------------
% Please supply a list of author last names for the page header. in
% one of these formats:
%
% EXAMPLES:
% \authormark{Lastname}
% \authormark{Lastname1 \& Lastname2}
% \authormark{Lastname1, Lastname2, ... \& LastnameN}
% \authormark{Lastname et al.}
%
% Use the "et al." form in the case of seven or more authors, or if
% the preferred form is too long to fit in the header.

\authormark{Currie}

%-----------------------------------------------------------------------
%			Subject Index keywords
%-----------------------------------------------------------------------
% Enter a comma separated list of up to 6 keywords describing your
% paper.  These will NOT be printed as part of your paper; however,
% they will be used to generate the subject index for the proceedings.
% There is no standard list; however, you can consult the indices
% for past proceedings (http://adass.org/adass/proceedings/).
%
% EXAMPLE:  \keywords{visualization, astronomy: radio, parallel
%                     computing, AIPS++, Galactic Center}
%
% In this example, the author noticed that "radio astronomy" appeared
% in the ADASS VII Index as "astronomy" being the major keyword and
% "radio" as the minor keyword.  The colon is used to introduce another
% level into the index.

\keywords{pipelines, data: reduction, FITS, Perl, Starlink}

%-----------------------------------------------------------------------
%			       Abstract
%-----------------------------------------------------------------------
% Type abstract in the space below.  Consult the User Guide and Latex
% Information file for a list of supported macros (e.g. for typesetting 
% special symbols). Do not leave a blank line between \begin{abstract} 
% and the start of your text.

\begin{abstract}          % Leave intact
ORAC-DR---a flexible reduction pipeline---was originally developed by
the Joint Astronomy Centre for real-time inspection of reduced data at
its telescopes.  Starlink is extending ORAC-DR to process at home
institutions data from other observatories, notably ESO, whose
instruments make no provision for ORAC-DR.  I outline the problems
encountered and solutions implemented or proposed to apply ORAC-DR to
the infra-red instruments ISAAC, NACO, INGRID, and Classic~Cam.
\end{abstract}

%-----------------------------------------------------------------------
%			      Main Body
%-----------------------------------------------------------------------
% Place the text for the main body of the paper here.  You should use
% the \section command to label the various sections; use of
% \subsection is optional.  Significant words in section titles should
% be capitalized.  Sections and subsections will be numbered
% automatically. 
%
% EXAMPLE:  \section{Introduction}
%           ...
%           \subsection{Our View of the World}
%           ...
%           \section{A New Approach}
%
% It is recommended that you look at the sample papers, sample1.tex
% and sample2.tex, for examples for formatting references, footnotes,
% figures, equations, html links, lists, and other special features.  

\section{Introduction}

\htmladdnormallink{ORAC-DR}{http://www.jach.hawaii.edu/JACpublic/UKIRT/software/oracdr/}
is an intelligent, data-driven, tailorable pipeline system,
developed at the \htmladdnormallink{Joint Astronomy Centre}
{http://www.jach.hawaii.edu/JACpublic/index.html} (JAC).
Object-oriented Perl data-reduction recipes use Starlink applications
to process the bulk data, invoked within `primitives', each of which
correspond to an astronomically meaningful step in the reduction.  It
has general code for common operations such as managing calibrations,
flexible data display, and data-format conversions.

Before the work described here, ORAC-DR already supported a wide
variety of instruments and techniques at 
\htmladdnormallink{UKIRT}{http://www.jach.hawaii.edu/JACpublic/UKIRT/},
\htmladdnormallink{JCMT}{http://www.jach.hawaii.edu/JACpublic/JCMT/}
and \htmladdnormallink{AAO}{http://www.aao.gov.au/} (Cavanagh et al.
2002 and references therein).  At these institutions the data
collection co-operates with ORAC-DR, so the sequence of integrations
match the reduction recipes, and ORAC-DR-specific metadata are
present.

\htmladdnormallink{Starlink}{http://www.starlink.ac.uk} realised that ORAC-DR had the potential to reduce data from
telescopes even without this symbiosis, or face-to-face dialogues with
instrument scientists, available in an observatory setting.  This is
particularly important for UK astronomers recently exposed to complex
ESO instruments.  It has potential for Virtual Observatory (VO)
applications, where an archive stores the raw data and the astronomer
wants a reduced mosaic or fully calibrated spectrum.  The reduction
recipes can encapsulate knowledge of the instrument signature and
reduction techniques, freeing the astronomer to concentrate on the
science.

This paper describes some of the problems and solutions adopted to
apply ORAC-DR to some infra-red instruments
\htmladdnormallink{ISAAC}{http://www.eso.org/instruments/isaac/} and 
\htmladdnormallink{NACO}{http://www.eso.org/instruments/naco/} at
\htmladdnormallink{ESO}{http://www.eso.org/};
\htmladdnormallink{INGRID}{http://www.ing.iac.es/Astronomy/instruments/ingrid/}
at the \htmladdnormallink{ING, La
Palma}{http://www.ast.cam.ac.uk/ING/PR/index.html}; and 
\htmladdnormallink{Classic~Cam}{http://www.ociw.edu/lco/magellan/instruments/Classic_Cam/}
on \htmladdnormallink{Magellan}{http://www.ociw.edu/magellan/}.

\section{Problems and Solutions}
\subsection{Metadata}

ORAC-DR is driven by FITS metadata.  These comprise steering headers
like the observation and group numbers, and recipe name; and
recipe-specific metadata, such as filter, exposure time, and grating
name.  ORAC-DR translates the latter type of header, sometimes in
combination, into common internal headers with standard meanings and
units, thereby insulating the recipes from instrument-specific
headers.

While the observation number is usually present, the other steering
headers are not.  As ORAC-DR works sequentially, preprocessor C-shell
scripts determine from a combination of headers or assumptions based
upon web-based information, the delineation of the groups (each group
being a related set of observations).  For example, the
{\tt HIERARCH.ESO.TPL.EXPNO = 1} in ESO data implies the start of a group;
as does when any of the main attributes like filter, and exposure time
change in Classic~Cam headers.  The scripts insert the few required
steering keywords into the metadata.

There is a close mapping of ESO templates to existing ORAC-DR recipes.
However, it's not always one-to-one, for example it's not clear
whether to self-flat or use a separate flat for a dithered infra-red
sequence.  These are different recipes, but the same template.  The
user can always substitute the appropriate recipe on the ORAC-DR
command line, or edit the {\tt RECIPE} header.  Commonality between
dictionaries at ESO, does permit general infrastructure, subclassing
where necessary to specific instruments.  Code reuse is an ORAC-DR
mantra.

The quality of FITS headers is extremely variable, and many cases omit
vital data for recipes, let alone provide an accurate record of the
instrument status and something at all adequate for VO use. The
Classic~Cam headers were particularly spartan.  Even the detailed ESO
headers had some metadata needed by the recipes missing, such as the
grating dispersion.  Appropriate values were usually found in or
derived from web pages and manuals, which for ISAAC were very good.  If
only other observatories were as diligent.  However, occasionally I had
to consult an instrument scientist, more so for Classic~Cam and INGRID.
Dialogue also ensued whenever there were two headers ostensibly
presenting the same attribute, but with different value, or the
meaning of a header was unclear or its units were not specified.

Headers change as experience with an instrument grows and a pipeline
must cope.  Where available, data dictionaries help greatly, but their
history is not readily available to the external developer.  It would
help if the CVS repositories of the dictionaries were made publically
accessible.  For ESO data I extended the translations empirically, by
testing with data over a wide range of observation dates.  Judging by
the experience at UKIRT, no doubt the existing translations are not
comprehensive and will require more exposure to users' data over an
extended period.  Some expected headers were not always present, so
the translations must be flexible and search a few headers in turn.
The data dictionaries don't include the dependencies.

The ESO headers include hierarchical keywords, which required
modification of the Perl package \htmladdnormallink{{\tt
Astro::FITS::Header}}{http://search.cpan.org/~aallan/}.
Starlink applications had already handled these headers for many years.

\subsection{File Naming}

ORAC-DR file naming mostly follows the JAC conventions, although there
is some provision for others.  ESO use the UT date and time in
one of its naming conventions, while ORAC-DR currently needs the
observation number.  Thus the preprocessor scripts rename the files to
have JAC-like nomenclature, while allowing for times spanning midnight UT.
A goal is for ORAC-DR to accept such names, and to allow selection of
observations made between certain times.

\subsection{Test Data}

Some of the ports were instigated by users with data to reduce.  Data
from an individual user is highly selective.  There may be no clues as
to how representative they are.  The INGRID, Classic~Cam, and NACO
pipelines therefore only have limited testing.  NACO does, however,
benefit from the ISAAC pipeline.  

Accessing data from the ESO archive is not straightforward when you
do not know which object you want, and whether the data you select is
representative or pretty.  This demands closer co-operation with the
instrument scientists and archivists.  It would be useful to collate
representative test data for regression testing of ORAC-DR.

\subsection{Instrumental Idiosyncrasies}

Every detector and instrument has its own properties.  For example,
ISAAC has spatial distortion, electronic ghosting, variable bias.
NACO---at least in the data seen thus far---has large swathes of very
noisy pixels.  INGRID has a multi-extension FITS file containing
pre-exposure and post-exposure images.  Each instrument has a
different non-linearity correction.  Polarimetry masks and dither
patterns vary between instruments.

It is easy to insert new steps into a recipe or have
instrument-specific steps without affecting existing primitives or
other instruments' reduction.  It's then a question of finding a
suitable algorithm.  All were solved using existing atomic Starlink
applications.  While the solutions were not especially sophisticated,
they seem fine judging by the photometric accuracy and quality
achieved.

\section{Recipe Development}

The processing demands of the instruments new to ORAC-DR required new
recipe code, which in some cases benefited the already supported
instruments.  Nevertheless much primitive code was reused.  The
highlights of these are listed below.

\begin{itemize}
\item Automatic wavelength calibration of non-thermal infra-red ISAAC
spectra.
\item Support for dithered target and offset spectral beams.
\item New recipes for calibration-lamp flats.
\item Combine multiple dark and arc integrations, and process
multi-filter groups of flat-field frames.
\item Correction for spatial distortion, variable bias, and electronic
ghosting.
\item Cope with several more observation sequences, such as various
object-sky patterns for chop-sky imaging, not just {\em SOSOS}\ldots and
uses time-based interpolation for its modal sky subtraction.
\end{itemize}

\section{Conclusions}
\begin{itemize}

\item It is viable to generate an ORAC-DR pipeline producing publication or
near-publication quality results for non-ORAC observatories.  Most
recipes were used `off the shelf'.

\item It is possible to achieve this in days or weeks rather than months or
years in the simpler cases.  The exact time depends on the number of
new recipes needed, the quality of the metadata, the instrument
complexity and special characteristics.  To fully support an
instrument like ISAAC in all its modes is still about six months' work,
but extending to other ESO infra-red instruments will be much less.
As the recipes encompass more instruments there will be fewer new
problems to tackle.

\item Comprehensive testing is not possible for the external programmer.  Fine
tuning for secular changes, and rare or pathological cases relies on user
feedback.

\item Standards for metadata need to improve markedly at certain institutions.
This includes providing clear comments and units.  A data dictionary is
desirable too, preferably with interdependencies included.  The dilettante
culture must change for the VO too.  Time to name and shame!

\item It is better for the metadata to be too verbose, and record more
instrumental and sequence data than is sufficient.  Such information can also
be useful diagnostics to secular or sudden changes of instrument properties.
The pipeline can only take account of these, if it can recognise them.
If observatories could be persuaded to add a few additional steering
headers, the pre-processing step could be avoided.  ORAC-DR offers a cheap
pipe\-lining system for observatories without resources to create their own.

\item Astronomers do not like black boxes, and will surely wish to
tinker with the provided recipes.  While ORAC-DR offers some tools for
this, it needs a configuration system and command-line overrides for
recipe variables to permit easier tailoring of recipes by astronomers.
This is in development.  The ORAC-DR programmers' manual (Jenness \&
Economou 2001) needs updating and made astronomer friendly.

\item It would be useful to have collated infra-red standard-star data
available from data centres, accessed by cgi scripts, as already happens
for spectroscopy calibrations in ORAC-DR, or via a web service.
\end{itemize}

\acknowledgments

Starlink is funded by the Particle Physics and Astronomy Research
Council (PPARC) and managed by the Space Science and Technology
Department (SSTD) of the Central Laboratory of the Research Councils
(CLRC).


%-----------------------------------------------------------------------
%			      References
%-----------------------------------------------------------------------
% List your references below within the reference environment
% (i.e. between the \begin{references} and \end{references} tags).
% Each new reference should begin with a \reference command which sets
% up the proper indentation.  Observe the following order when listing
% bibliographical information for each reference:  author name(s),
% publication year, journal name, volume, and page number for
% articles.  Note that many journal names are available as macros; see
% the User Guide listing "macro-ized" journals.   
%
% EXAMPLE:  \reference Hagiwara, K., \& Zeppenfeld, D.\  1986, 
%                Nucl.Phys., 274, 1
%           \reference H\'enon, M.\  1961, Ann.d'Ap., 24, 369
%           \reference King, I.\ R.\  1966, \aj, 71, 276
%           \reference King, I.\ R.\  1975, in Dynamics of Stellar 
%                Systems, ed.\ A.\ Hayli (Dordrecht: Reidel), 99
%           \reference Tody, D.\  1998, \adassvii, 146
%           \reference Zacharias, N.\ \& Zacharias, M.\ 2003,
%                \adassxii, \paperref{P7.6}
% 
% Note the following tricks used in the example above:
%
%   o  \& is used to format an ampersand symbol (&).
%   o  \'e puts an accent agu over the letter e.  See the User Guide
%      and the sample files for details on formatting special
%      characters.  
%   o  "\ " after a period prevents LaTeX from interpreting the period 
%      as an end of a sentence.
%   o  \aj is a macro that expands to "Astron. J."  See the User Guide
%      for a full list of journal macros
%   o  \adassvii is a macro that expands to the full title, editor,
%      and publishing information for the ADASS VII conference
%      proceedings.  Such macros are defined for ADASS conferences I
%      through XI.
%   o  When referencing a paper in the current volume, use the
%      \adassxii and \paperref macros.  The argument to \paperref is
%      the paper ID code for the paper you are referencing.  See the 
%      note in the "Paper ID Code" section above for details on how to 
%      determine the paper ID code for the paper you reference.  
%
\begin{references}
\reference Cavanagh, B., Hirst, P., Jenness, T., Economou, F.,
    Currie, M.\ J., Todd, S., \& Ryder, S.\ D.\ 2003 \adassxii, 237
\reference Jenness, T.\ \& Economou, F.\ 2001, \htmladdnormallink{Starlink User 
Note 233}{http://http://www.starlink.rl.ac.uk/star/docs/sun233.htx/sun233.html},
Starlink Project, CCLRC
\end{references}

% Do not place any material after the references section

\end{document}  % Leave intact
