%ADASS_PROCEEDINGS_FORM%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% SAMPLE1.TEX -- ADASS XII (2002) ASP Conference Proceedings sample
% paper with minimal markup. Based on the sample from ADASS XI (01).
%
% This is a simple example.  If you want to see a more comprehensive
% sample paper,  take a look at sample2.tex.
%
% Much of the input will be enclosed by braces (i.e., { }).  The
% percent sign, "%", denotes the start of a comment; text after it
% will be ignored by LaTeX.  You might also notice in some of the
% examples below the use of "\ " after a period; this prevents LaTeX
% from interpreting the period as the end of a sentence and putting
% extra space after it.   
% 
% You should check your paper by processing it with LaTeX.  For
% details about how to run LaTeX as well as how to print out the User
% Guide, consult the README file.  
%
% If you do not have access to the LaTeX software or a laser printer
% at your site, you can still prepare your paper following the
% instructions in the User Guide.  In such cases, the editors will
% process the file and make any necessary editorial adjustments.
% 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% 
\documentclass[11pt,twoside]{article}  % Leave intact
\usepackage{adassconf}

% If you have the old LaTeX 2.09, and not the current LaTeX2e, comment
% out the \documentclass and \usepackage lines above and uncomment
% the following:

%\documentstyle[11pt,twoside,adassconf]{article}

\begin{document}   % Leave intact

%-----------------------------------------------------------------------
%                           Paper ID Code
%-----------------------------------------------------------------------
% Enter the proper paper identification code.  The ID code for your
% paper is the session number associated with your presentation as
% published in the official conference proceedings.  You can
% find this number locating your abstract in the printed proceedings
% that you received at the meeting or on-line at the conference web
% site; the ID code is the letter/number sequence proceeding the title 
% of your presentation.  
%
% This will not appear in your paper; however, it allows different
% papers in the proceedings to cross-reference each other.  Note that
% you should only have one \paperID, and it should not include a
% trailing period.
%

\paperID{P8-5}
%%%% ID=P8-5

%-----------------------------------------------------------------------
%                           Paper Title 
%-----------------------------------------------------------------------
% Enter the title of the paper.
%
% EXAMPLE: \title{A Breakthrough in Astronomical Software Development}
%
% If your title is so long as to fill the page header when you print it,
% then please supply a short form as a \titlemark.
%
% EXAMPLE:
%  \title{Rapid Development for Distributed Computing, with Implications
%         for the Virtual Observatory}
%  \titlemark{Rapid Development for Distributed Computing}
%

\title{Efficient usage of HPC horsepower for the LOFAR telescope}
%\titlemark{ }

%-----------------------------------------------------------------------
%                         Authors of Paper
%-----------------------------------------------------------------------
% Enter the authors followed by their affiliations.  The \author and
% \affil commands may appear multiple times as necessary.  List each
% author by giving the first name or initials first followed by the
% last name.  Authors with the same affiliations should grouped
% together. 
%
% Try to limit the front matter to no more than three \author
% commands.  Group authors with the same affiliations.  Too many
% \author commands fills the first page of the paper with little
% actual text.

\author{Kjeld v.d. Schaaf}
\affil{ASTRON, Postbus 2, 7990 AA Dwingeloo, The Netherlands}

%-----------------------------------------------------------------------
%                        Contact Information
%-----------------------------------------------------------------------
% This information will not appear in the paper but will be used by
% the editors in case you need to be contacted concerning your
% submission.  Enter your name as the contact along with your email
% address.

\contact{Kjeld v.d. Schaaf}
\email{schaaf@astron.nl}

%-----------------------------------------------------------------------
%                     Author Index Specification
%-----------------------------------------------------------------------
% Specify how each author name should appear in the author index.  The 
% \paindex{ } should be used to indicate the primary author, and the
% \aindex for all other co-authors.  You MUST use the following
% syntax: 
%
% SYNTAX:  \aindex{LASTNAME, F. M.}
% 
% where F is the first initial and M is the second initial (if
% used).  This guarantees that authors that appear in multiple papers
% will appear only once in the author index.  
%
% EXAMPLE: \paindex{Crabtree, D.}
%          \aindex{Manset, N.}
%          \aindex{Veillet, C.}
%
% NOTE: this information is also used to build the author list that
% appears in the table of contents.  Authors will be listed in the order
% of the \paindex and \aindex commmands.
%

\paindex{Schaaf, Kjeld v.d.}

%-----------------------------------------------------------------------
%                     Author list for page header
%-----------------------------------------------------------------------
% Please supply a list of author last names for the page header. in
% one of these formats:
%
% EXAMPLES:
% \authormark{LASTNAME}
% \authormark{LASTNAME1 \& LASTNAME2}
% \authormark{LASTNAME1, LASTNAME2, ... \& LASTNAMEn}
% \authormark{LASTNAME et al.}
%
% Use the "et al." form in the case of seven or more authors, or if
% the preferred form is too long to fit in the header.

\authormark{Schaaf} %% \authormark{Schaaf, Kjeld v.d.} -- Mod. FO 22.04.04

%-----------------------------------------------------------------------
%                       Subject Index keywords
%-----------------------------------------------------------------------
% Enter up to 6 keywords describing your paper.  These will NOT be
% printed as part of your paper; however, they will be used to
% generate the subject index for the proceedings.  There is no
% standard list; however, you can consult the indices for past ADASS
% proceedings (http://adass.org/adass/proceedings/).

\keywords{HPC, LOFAR, cluster, MPI, parallel, software: architecture}

%-----------------------------------------------------------------------
%                              Abstract
%-----------------------------------------------------------------------
% Type abstract in the space below.  Consult the User Guide and Latex
% Information file for a list of supported macros (e.g. for typesetting 
% special symbols). Do not leave a blank line between \begin{abstract} 
% and the start of your text.

\begin{abstract}          % Leave intact
The operation of novel phased array based radio telescopes requires
vast amount of computational power for the processing of observed
data. In the LOFAR telescope, the central on-line and off-line systems
will use HPC resources provided by a 1000+ nodes cluster computer. In
order to profit from the enormous computational power of this machine
an application development framework is built. This framework support
the transformation of the current single-node data reduction
applications to parallel equivalents that can execute on large
clusters. In the online systems, real-time signal processing pipelines
can be executed. Furthermore, the complexity of controlling many
concurrently running observations and analyses on an inhomogeneous
hardware set is tackled.
\end{abstract}

%-----------------------------------------------------------------------
%                             Main Body
%-----------------------------------------------------------------------
% Place the text for the main body of the paper here.  You should use
% the \section command to label the various sections; use of
% \subsection is optional.  Significant words in section titles should
% be capitalized.  Sections and subsections will be numbered
% automatically. 

\section{The LOFAR Central Processor Facility}

The central processor of LOFAR should provide the necessary processing
and storage capacity for the applications running on it. Although
multiple types of sensors are attached to the LOFAR infrastructure
that provide data for multiple scientific fields, we focus here on the
radio astronomical data. For these applications, we can identify three
key processing types:
\begin{itemize}
\item For sky imaging applications, the central processor is used for
    correlation, beamforming, integration and, after short term
    storage, calibration and image creation.

\item For pulsar observations, an ``extended core'' of up to 50\% 
of the stations produce data for a central beamformer. A major
processing task is the de-dispersion measuring.

\item For transient detection and analysis of time scales of a
second or more, series of snapshot images can be analysed using
Fourier transforms. For shorter time scales, detection algorithms
operate directly on the station beams or antenna data. 

\end{itemize}

\noindent	%% Added FO 22.04.04
The aim of the central processor architecture is to provide first of
all enough processing power, data transport bandwidth and storage
capacity to allow for continuous operation of the observation
facility. Secondly, the central processor will be built as a
signal-processing cluster aimed at multiple types of data processing. 

The hardware architecture of the central processor is based on cluster computer techniques
with hybrid cluster nodes and a high bandwidth switch fabric
interconnect system. The different types of cluster nodes represent
the required “flavours” of processing capacity. We have for instance
nodes based on a normal workstation to which digital signal processing
boards are added, containing system on chip processors such as a FPGA
with a DSP IP core.  Another “flavour” of nodes is for example the
storage node, which could be a workstation, connected to a
RAID disk array.



 
The central processor facility is constructed as a series of planes
with high bandwidth point-to-point connections between the planes,
thus forming pipelines. Orthogonal to the pipeline direction, an
all-to-all network topology is present which is used to bring data
streams to the correct pipeline. Especially in the first planes, such
routing capacity is needed for re-ordering the input data. For example
in the case of imaging applications for astronomy, a complete matrix
transpose of all input data is required, which translates to an
all-to-all data transport tasks on the first one or two planes in the
cluster.

The correlator architecture of LOFAR requires a transpose operation on
all input signals at the central site. With 100  stations each
providing 32000 spectral channels, this is a gigantic routing task. In
the central processor design this transpose function is executed in
two steps. In the first step, packages of multiple frequency channels
are routed in the first two planes of the cluster. The fine-grained
transpose operation that remains to be performed can efficiently be
executed by the node processors in cache memory. The transpose
operation requires a bandwidth of  ~1Gbps between all nodes in a
plane. Other applications may use the in-plane routing capacity to
distribute large processing tasks over multiple pipelines. 

The combination of pipeline connections and fully routed planes is
designed as one large switch fabric, although different technologies
might be used for the planes and pipeline connections. 

The combination of fully routed planes and pipelines being fed by
those planes offers a completely scalable hardware structure. The
available processing power can easily be distributed over multiple
applications running in parallel. More pipelines can be added to the
switch fabric operating as data farmer.

In the processing pipeline a few planes are organised as storage
system. This section of the cluster is responsible for the storage of
complete measurements until the observation data is fully processed
and the final data products are produced. This provides the
transformation of streaming data into a coherent dataset. Since the
data analysis applications are off-line processes, storage facilities
for several days are supplied. The required storage capacity of order
500 TByte can be implemented in two planes containing storage servers,
each equipped with RAID disk arrays. 

The central processor design is based on abstractions from available
techniques to avoid dependency on actual available
implementations. This is very important to be able to fully profit
from Moore's law.


\section{Programming Model}

A programming model has been designed in which the applications can be
expressed and mapped onto the available hardware. The programming
model is provided through an application development library layer
built on the middleware layer. The application
is modelled in pipes and filter style using a hierarchy of processing
steps that implement functional behaviour. The processing steps are
connected to each other using an abstraction of the message passing
paradigm, thus embedding parallel programming in the framework.

The scientific applications are expressed in functional steps in the
process step hierarchy. This whole hierarchy, along with the
functional connections is programmed into a single set of source
files. The source files can then be mapped onto the available hardware
in multiple ways. During the mapping, the exact processing hardware
for the processing steps are selected and the connection
implementations are chosen. The application source code can be mapped
onto one ore more interacting processes. Each such application has its
own virtual machine responsible for the correct sequential execution
of the application steps. Since the virtual
machines in all applications are generated from the same source file,
the synchronisation between the applications is guaranteed.

The processing model supports multiple message passing
implementations. Currently, implementations have been tested for
MPICH, ScaMPI, ShMem, Memcpy and Corba. 

The Corba implementation uses remote invocations for the actual data
transport. Cyclic input and output buffers are used together with
event handlers in separate threads in order to get the required
blocking behaviour for the programming model. Performance tests have
shown similar bandwidth characteristics for the MPICH and Corba
implementations. 

The signal processing boards added to part of the cluster nodes are
controlled by a middleware layer that is currently being developed by
the project members. This middleware layer provides the appropriate
hardware abstractions required by the application development
framework. This also allows for adoption of the new hardware items
into the monitoring and control system.

 Note that the programming model, and therefore the application code,
 is independent on the data transport implementation used since only a
 small subset from the message passing paradigm is needed. Other
 middleware implementations can be used for the connections. Also
 direct device-to-device connections can be controlled from the
 connection abstraction in the model.

The multiple message passing implementations supported in the
programming model can also be mixed. A demonstrator has been built in
which a ScaMPI application performs data routing tasks and sends its
data to multiple processing pipelines implemented with MPICH
communication. The data transport between the ScaMPI and MPICH
programs is made by a Corba connection.

The application development platform is optimised to minimise
overhead. The data transport mechanisms in the framework are based on
existing middleware implementations were feasible. Dedicated transport
mechanisms are implemented were needed. There the data transport bandwidth for data transport between
processes on a SMP node is measured.

The mapping of the application's processing steps on to the hardware
is made independent of the processing step's implementation. Therefore
the same application can be mapped onto the hardware in multiple
ways. This is used for performance optimisation by mapping of the
processing blocks on the appropriate "flavour" of cluster nodes. The
strategy pattern is used to select the correct executable for the
target processor. 

The switching fabric can be operated by a data farming process that
farms out blocks of data to pipelined processes. This data farming
process is used for both load balancing and fault tolerance. The fault
tolerant operation is demonstrated using the Corba connector in
combination with Corba control methods added to the processing
steps. The pipeline applications are now both controlled and fed with
data through Corba objects.


%-----------------------------------------------------------------------
%                             References
%-----------------------------------------------------------------------
% List your references below within the reference environment
% (i.e. between the \begin{references} and \end{references} tags).
% Each new reference should begin with a \reference command which sets
% up the proper indentation.  Observe the following order when listing
% bibliographical information for each reference:  author name(s),
% publication year, journal name, volume, and page number for
% articles.  Note that many journal names are available as macros; see
% the User Guide for a listing "macro-ized" journals.   
%
% Note the following are some of the tricks that can be used:
%
%   o  \& is used to format an ampersand symbol (&).
%   o  \'e puts an accent grave over the letter e.  See the User Guide
%      for details on formatting special characters.  
%   o  "\ " after a period prevents LaTeX from interpreting the period 
%      as an end of a sentence.
%   o  \aj is a macro that expands to "Astron. J."  See the User Guide
%      for a full list of journal macros
%   o  \adassvii is a macro that expands to the full title, editor,
%      and publishing information for the ADASS VII conference
%      proceedings.  Such macros are defined for ADASS conferences I
%      through IX.
%   o  When referencing a paper in the current volume, use the
%      \adassix and \paperref macros.  The argument to \paperref is
%      the paper ID code for the paper you are referencing.  See the 
%      note in the "Paper ID Code" section above for details on how to 
%      determine the paper ID code for the paper you reference.  
%

%  [5]     L.P.Huse et al., "ScaMPI - Design and implementation", http://www.scali.com/whitepaper/other/scampidesign.pdf
% [6]     P. Geoffray, "OPIOM: Off-Processor IO with Myrinet", Proceedings of the First IEEE/ACM International Symposium on Cluster Computing and the Grid, Brisbane, May 2001, pp. 261-268.

\begin{references}
\reference Achterop, S., 2001, Proceedings of ProRISC 
\reference Tessier, R., \& Burleson, W., 2001, Journal of VLSI Signal
Processing, vol.\ 28 , 7 , 27
\reference Vos, C.M. de, \ Schaaf, K. v.d. \& Bregman, J.D., 2001 , Proceedings of the First IEEE/ACM International Symposium
on Cluster Computing and the Grid, Brisbane, 156, 160
\end{references}

% Do not place any material after the references section

\end{document}  % Leave intact
