%APN3_PROCEEDINGS_FORM%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% TEMPLATE.TEX -- APN3 (2003) ASP Conference Proceedings template.
%
% Derived from ADASS VIII (98) ASP Conference Proceedings template
% Updated by N. Manset for ADASS IX (99), F. Primini for ADASS 2000,
% D.Bohlender for ADASS 2001, and H. Payne for ADASS XII and LaTeX2e.
%
% Use this template to create your proceedings paper in LaTeX format
% by following the instructions given below.  Much of the input will
% be enclosed by braces (i.e., { }).  The percent sign, "%", denotes
% the start of a comment; text after it will be ignored by LaTeX.  
% You might also notice in some of the examples below the use of "\ "
% after a period; this prevents LaTeX from interpreting the period as
% the end of a sentence and putting extra space after it.  
% 
% You should check your paper by processing it with LaTeX.  For
% details about how to run LaTeX as well as how to print out the User
% Guide, consult the README file.  You should also consult the sample
% LaTeX papers, sample1.tex and sample2.tex, for examples of including
% figures, html links, special symbols, and other advanced features.
%
% If you do not have access to the LaTeX software or a laser printer
% at your site, you can still prepare your paper following the
% instructions in the User Guide.  In such cases, the editors will
% process the file and make any necessary editorial adjustments.
% 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% 
\documentclass[11pt,twoside]{article}  % Leave intact
\usepackage{adassconf}

% If you have the old LaTeX 2.09, and not the current LaTeX2e, comment
% out the \documentclass and \usepackage lines above and uncomment
% the following:

%\documentstyle[11pt,twoside,adassconf]{article}

\begin{document}   % Leave intact

%-----------------------------------------------------------------------
%			    Paper ID Code
%-----------------------------------------------------------------------
% Enter the proper paper identification code.  The ID code for your
% paper is the session number associated with your presentation as
% published in the official conference proceedings.  You can           
% find this number locating your abstract in the printed proceedings
% that you received at the meeting or on-line at the conference web
% site; the ID code is the letter/number sequence proceeding the title 
% of your presentation. 
%
% This will not appear in your paper; however, it allows different
% papers in the proceedings to cross-reference each other.  Note that
% you should only have one \paperID, and it should not include a
% trailing period.
%
% EXAMPLE: \paperID{O4-1}
% EXAMPLE: \paperID{P7-7}
%

\paperID{P10-16}
%%%% ID=P10-16

%-----------------------------------------------------------------------
%		            Paper Title 
%-----------------------------------------------------------------------
% Enter the title of the paper.
%
% EXAMPLE: \title{A Breakthrough in Astronomical Software Development}
% 
% If your title is so long as to fill the page header when you print it,
% then please supply a short form as a \titlemark.
%
% EXAMPLE: 
%  \title{Rapid Development for Distributed Computing, with Implications
%         for the Virtual Observatory}
%  \titlemark{Rapid Development for Distributed Computing}
%

\title{The HST BestRef System - Determining the proper calibration reference files for an HST exposure}
\titlemark{The HST BestRef System}

%-----------------------------------------------------------------------
%		          Authors of Paper
%-----------------------------------------------------------------------
% Enter the authors followed by their affiliations.  The \author and
% \affil commands may appear multiple times as necessary (see example
% below).  List each author by giving the first name or initials first
% followed by the last name.  Authors with the same affiliations
% should grouped together. 
%
% EXAMPLE: \author{Raymond Plante, Doug Roberts, 
%                  R.\ M.\ Crutcher\altaffilmark{1}}
%          \affil{National Center for Supercomputing Applications, 
%                 University of Illinois Urbana-Champaign, Urbana, IL
%                 61801}
%          \author{Tom Troland}
%          \affil{University of Kentucky}
%
%          \altaffiltext{1}{Astronomy Department, UIUC}
%
% In this example, the first three authors, "Plante", "Roberts", and
% "Crutcher" are affiliated with "NCSA".  "Crutcher" has an alternate 
% affiliation with the "Astronomy Department".  The fourth author,
% "Troland", is affiliated with "University of Kentucky"

\author{Michael S.\ Swam, Steve Lubow, Lee Hurt}
\affil{Space Telescope Science Institute, 3700 San Martin Drive,
    Baltimore, MD 21218}

%-----------------------------------------------------------------------
%			 Contact Information
%-----------------------------------------------------------------------
% This information will not appear in the paper but will be used by
% the editors in case you need to be contacted concerning your
% submission.  Enter your name as the contact along with your email
% address.
% 
% EXAMPLE:  \contact{Dennis Crabtree}
%           \email{crabtree@cfht.hawaii.edu}
%

\contact{Michael Swam}
\email{mswam@stsci.edu}

%-----------------------------------------------------------------------
%		      Author Index Specification
%-----------------------------------------------------------------------
% Specify how each author name should appear in the author index.  The 
% \paindex{ } should be used to indicate the primary author, and the
% \aindex for all other co-authors.  You MUST use the following
% syntax: 
%
% SYNTAX:  \aindex{Lastname, F. M.}
% 
% where F is the first initial and M is the second initial (if
% used).  This guarantees that authors that appear in multiple papers
% will appear only once in the author index.  
%
% EXAMPLE: \paindex{Crabtree, D.}
%          \aindex{Manset, N.}        
%          \aindex{Veillet, C.}        
%
% NOTE: this information is also used to build the author list that
% appears in the table of contents.  Authors will be listed in the order
% of the \paindex and \aindex commmands.
%

\paindex{Swam, M. S.}
\aindex{Lubow, S.}
\aindex{Hurt, L.}

%-----------------------------------------------------------------------
%		      Author list for page header	
%-----------------------------------------------------------------------
% Please supply a list of author last names for the page header. in
% one of these formats:
%
% EXAMPLES:
% \authormark{Lastname}
% \authormark{Lastname1 \& Lastname2}
% \authormark{Lastname1, Lastname2, ... \& LastnameN}
% \authormark{Lastname et al.}
%
% Use the "et al." form in the case of seven or more authors, or if
% the preferred form is too long to fit in the header.

\authormark{Swam, Lubow, \& Hurt}

%-----------------------------------------------------------------------
%			Subject Index keywords
%-----------------------------------------------------------------------
% Enter a comma separated list of up to 6 keywords describing your
% paper.  These will NOT be printed as part of your paper; however,
% they will be used to generate the subject index for the proceedings.
% There is no standard list; however, you can consult the indices
% for past proceedings (http://adass.org/adass/proceedings/).
%
% EXAMPLE:  \keywords{visualization, astronomy: radio, parallel
%                     computing, AIPS++, Galactic Center}
%
% In this example, the author noticed that "radio astronomy" appeared
% in the ADASS VII Index as "astronomy" being the major keyword and
% "radio" as the minor keyword.  The colon is used to introduce another
% level into the index.

\keywords{HST, calibration, databases, STScI, Python, reference file}

%-----------------------------------------------------------------------
%			       Abstract
%-----------------------------------------------------------------------
% Type abstract in the space below.  Consult the User Guide and Latex
% Information file for a list of supported macros (e.g. for typesetting 
% special symbols). Do not leave a blank line between \begin{abstract} 
% and the start of your text.

\begin{abstract}          % Leave intact
This paper will describe the software architecture of the 
Hubble Space Telescope (HST) BestRef 
system, which is used to determine the proper calibration reference files 
for a particular HST exposure.  The system, which makes use of the HST 
Calibration Database System (CDBS), is highly table-driven, allowing 
additions of new reference file types to support continual instrument 
calibration development as well as each new science instrument added 
during HST servicing missions.  The system also works in concert with 
the HST Archive Catalog database and its StarView and web-based user 
interfaces to supply archive users with the set of proper reference 
files for any exposure they might retrieve from the archive.
\end{abstract}

%-----------------------------------------------------------------------
%			      Main Body
%-----------------------------------------------------------------------
% Place the text for the main body of the paper here.  You should use
% the \section command to label the various sections; use of
% \subsection is optional.  Significant words in section titles should
% be capitalized.  Sections and subsections will be numbered
% automatically. 
%
% EXAMPLE:  \section{Introduction}
%           ...
%           \subsection{Our View of the World}
%           ...
%           \section{A New Approach}
%
% It is recommended that you look at the sample papers, sample1.tex
% and sample2.tex, for examples for formatting references, footnotes,
% figures, equations, html links, lists, and other special features.  

\section{The Purpose of HST BestRef}

The HST BestRef system serves two masters: the science data processing 
pipelines and the archive catalog.
The pre-archive pipeline receives science telemetry from HST and proceses 
the data into FITS files for the archive system (Swade, Hopkins, \& Swam 2001).
The On-The-Fly 
Reprocessing (OTFR) pipeline regenerates these science data products at 
archive retrieval time, by starting again from the base HST science 
telemetry (Swam, Hopkins, \& Swade 2001).  Both pipelines rely on 
BestRef to compute the best reference 
files for a given HST exposure so that these files can be 
applied during calibration.   The archive catalog relies on 
BestRef to perform updates to the stored list of best reference files held 
for each archived HST exposure.  When a science instrument team 
produces new reference files they often apply to some subset of the 
existing archive content.  This subset of exposures must be identified so 
that the new reference file can replace the out-dated recommendation in 
the catalog.  Using the BestRef system to serve both of these masters
ensures that both receive identical recommendations given the same inputs.

\section{The BestRef Architecture}

The major components of the BestRef software architecture are the CDBS,
the BestRef rules, and the BestRef rules engine.

\subsection{CDBS}

For every reference file created for a HST science instrument, a file 
description is installed in the CDBS (Cox, Lubow, \& Tullos, 1998).  
This description includes the 
instrument mode parameters (e.g. detector id, filter id, etc.) that 
define the types of exposures the reference file applies to.   This 
information is held in separate database tables for each HST science 
instrument, and each table contains fields for the union of all of the 
mode-selection parameters across that instrument's reference file set.
For any particular table row that documents one mode of a particular 
reference file type, there are likely some mode fields that do not apply 
to that type and these are given obvious phony values in that row.

Also part of the reference file description is a "use-after" date that 
marks the earliest date-time after which this reference 
file should be used.  When a delivery is made to the CDBS for a new 
reference file that exactly covers the same modes and 
use-after date as a previously delivered file, the previous file is 
marked "rejected".  The rejected file is not deleted from the database, 
which maintains historical tracking, but is flagged as inactive, marked
with the rejecting filename and date, and will 
never be recommended by BestRef again.
Given these detailed descriptions of reference file content and their
application modes, it is possible to write relatively simple database 
queries that
locate the proper reference file of a given type for a given HST exposure.

\subsection{The BestRef Rules}

All of the BestRef reference file selection rules for all HST instruments 
are stored in a file of XML.  These rules indicate the search criteria to 
use in selecting a particular reference file type for a particular HST 
science instrument.  They include file selection and file restriction 
rules, both of which can include conditionals (e.g. "only use central 
wavelength to select a STIS flat-field when in SPECTROSCOPIC mode", or 
"only use the STIS image distortion correction file in IMAGING mode").   
The rules also contain the mapping from FITS header keyword name to
the name of a particular reference file type, and a flag indicating 
if a reference file is required or optional (only failure to fill a required
file will generate an error condition).   The rules are read by the 
BestRef rules engine once at task start-up, and are held in memory as each 
selected science exposure has its reference files computed.   Here is a 
cutout example of the XML format for the STIS Image Distortion Correction 
table:

\footnotesize
\begin{verbatim}
<!--image distortion correction table -->
<REFFILE>
    <REFFILE_TYPE> IDC </REFFILE_TYPE>
    <REFFILE_KEYWORD> IDCTAB </REFFILE_KEYWORD>
    <REFFILE_FORMAT> TABLE </REFFILE_FORMAT>
    <FILE_SELECTION>
       <FILE_SELECTION_FIELD> DETECTOR </FILE_SELECTION_FIELD>
    </FILE_SELECTION>
    <RESTRICTION>
       <RESTRICTION_TEST> (aSource._keywords['OBSTYPE'][0] == 'IMAGING')
       </RESTRICTION_TEST>
    </RESTRICTION>
</REFFILE>
\end{verbatim}
\normalsize

Note the format of the "RESTRICTION-TEST" rule.  It is 
actually written in Python code, which is directly read and executed by 
the rules engine.  This provides flexibility for adding rules consisting 
of any Python construct that can be evaluated to a logical (True/False) 
value.  This same mechanism is used to add a "FILE-SELECTION-TEST" as shown
below, which 
only uses a field for file selection (e.g. CCDAMP) if certain conditions 
are met (e.g. the DETECTOR is NOT "SBC"), 
otherwise the field does not contribute to the selection criteria.

\footnotesize
\begin{verbatim}
<FILE_SELECTION>
  <FILE_SELECTION_FIELD> CCDAMP </FILE_SELECTION_FIELD>
  <FILE_SELECTION_TEST> (aSource._keywords['DETECTOR'][0] != 'SBC') 
  </FILE_SELECTION_TEST>
</FILE_SELECTION>
\end{verbatim}
\normalsize

By implementing
the reference file selection criteria in this rules format, it is quite
simple to add entries to support new reference file types or to alter
the selection criteria for existing types, and it is also possible to
dynamically build the CDBS queries in a table-driven manner.

\subsection{The BestRef Rules Engine}

The program that executes the BestRef rules is the rules engine.  This 
task consists of ~5100 lines of object-oriented Python code across 15 
modules.  The rules engine can obtain the rule inputs from either FITS
header keywords (in pipeline mode) or database fields
(in archive catalog mode).  The PyFITS Python FITS software 
(Hsu and Hodge 2004)
is used for the FITS interface, while the 
\htmladdnormallinkfoot{STPyDB module}
{http://www.stsci.edu/spst/UnixTransition/doc/stpydb.html} provides
a Python API to the Sybase databases at the Space Telescope Science
Institute (STScI).  Reference file
names are computed by applying the rules to the rule inputs and
generating database queries to the CDBS.  The resulting reference file
names can be stored in FITS keywords or written to
a file of SQL that is later applied the archive catalog.
The design choice of using rules and a rules engine allows new
reference file types that require standard rule matching to be added
without changing the rules engine.  Should selection of a particular
reference file type require a more complex algorithm, again no
change is needed to the rules engine as the design allows a rule
to call special case procedures, again written in Python, as shown
below:

\footnotesize
\begin{verbatim}
<REFFILE_FUNCTION> filename = cdbs_db.wfpc2_flatfile( reff, aSource) 
</REFFILE_FUNCTION>
\end{verbatim}
\normalsize

In this case, the WFPC-2 flatfield reference file requires both filter
ids in the selection query, but the filter order is irrelevant.  This
is handled internally by the special case procedure.  Currently only
three (3) special procedures are used throughout all of the rules
supporting the current HST instruments (ACS, STIS, NICMOS, WFPC-2) and the
next generation instruments (COS, WFC3), so the vast majority of
reference file types can be selected using standard rule matching.

\section{BestRef in use}

The flexibility of this architecture and implementation allows a single set 
of code to satisfy the various requirements for computing the set of 
reference files for a HST exposure.  In pipeline mode, the BestRef task is 
run just before calibration, and fills the FITS header reference file 
keywords with the names of the reference files to use during calibration.   
In archive catalog mode, a BestRef job is invoked periodically to read the 
list of newly delivered reference files from the CDBS and assign these 
new files to existing archived HST exposures.   When an archive user 
requests the set of best reference files be returned to them along with the 
reprocessed science data, database records in the archive catalog are read 
by the catalog user interface 
(\htmladdnormallinkfoot{StarView}{http://starview.stsci.edu/}
or
\htmladdnormallinkfoot{MAST}{http://archive.stsci.edu/})
to obtain the reference file recommendations filled by BestRef.

\section{Future Plans}

While the BestRef system is performing well in operations, some adjustments
are planned to address issues of timing.  As described, the archive catalog
BestRef is invoked on a periodic basis to look for new reference file
deliveries, but if such a delivery occurs long before the next periodic
invocation then contradictory recommendations can occur.  The OTFR system will
always return science exposures calibrated using the
most up-to-date set of reference files (which it obtains from the pipeline
BestRef), but the archive catalog can return an older set of reference
files to the user's disk since its tables have not yet had the benefit of
the periodic BestRef update.  Plans are to change the periodic update into
a direct invocation of the archive catalog BestRef when a new reference
file delivery occurs.

%-----------------------------------------------------------------------
%			      References
%-----------------------------------------------------------------------
% List your references below within the reference environment
% (i.e. between the \begin{references} and \end{references} tags).
% Each new reference should begin with a \reference command which sets
% up the proper indentation.  Observe the following order when listing
% bibliographical information for each reference:  author name(s),
% publication year, journal name, volume, and page number for
% articles.  Note that many journal names are available as macros; see
% the User Guide listing "macro-ized" journals.   
%
% EXAMPLE:  \reference Hagiwara, K., \& Zeppenfeld, D.\  1986, 
%                Nucl.Phys., 274, 1
%           \reference H\'enon, M.\  1961, Ann.d'Ap., 24, 369
%           \reference King, I.\ R.\  1966, \aj, 71, 276
%           \reference King, I.\ R.\  1975, in Dynamics of Stellar 
%                Systems, ed.\ A.\ Hayli (Dordrecht: Reidel), 99
%           \reference Tody, D.\  1998, \adassvii, 146
%           \reference Zacharias, N.\ \& Zacharias, M.\ 2003,
%                \adassxii, \paperref{P7.6}
% 
% Note the following tricks used in the example above:
%
%   o  \& is used to format an ampersand symbol (&).
%   o  \'e puts an accent agu over the letter e.  See the User Guide
%      and the sample files for details on formatting special
%      characters.  
%   o  "\ " after a period prevents LaTeX from interpreting the period 
%      as an end of a sentence.
%   o  \aj is a macro that expands to "Astron. J."  See the User Guide
%      for a full list of journal macros
%   o  \adassvii is a macro that expands to the full title, editor,
%      and publishing information for the ADASS VII conference
%      proceedings.  Such macros are defined for ADASS conferences I
%      through XI.
%   o  When referencing a paper in the current volume, use the
%      \adassxii and \paperref macros.  The argument to \paperref is
%      the paper ID code for the paper you are referencing.  See the 
%      note in the "Paper ID Code" section above for details on how to 
%      determine the paper ID code for the paper you reference.  
%
\begin{references}

\reference Cox, C.\ R., Lubow, S.\ \& Tullos, C.\ 1998, \spie{3349}, 218
\reference Hsu, J.\ C.\ \& Hodge, P.\ 2004, \adassxiii, \paperref{D4}
\reference Swade, D.\ A., Hopkins, E.\ \& Swam, M.\ S.\ 2001, \adassx, 295
\reference Swam, M.\ S., Hopkins, E.\ \& Swade, D.\ A.\ 2001, \adassx, 291

\end{references}

% Do not place any material after the references section

\end{document}  % Leave intact
