technical_implementation.html

<!DOCTYPE html>
<html  lang="en">
  <head>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width,initial-scale=1"><meta name="viewport" content="width=device-width, initial-scale=1" />

      <title>Technical implementation</title>
    
          <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
          <link rel="stylesheet" href="_static/theme.css " type="text/css" />
          <link rel="stylesheet" href="_static/custom.css" type="text/css" />
      
      <!-- sphinx script_files -->
        <script src="_static/documentation_options.js?v=5929fcd5"></script>
        <script src="_static/doctools.js?v=9a2dae69"></script>
        <script src="_static/sphinx_highlight.js?v=dc90522c"></script>

      
      <!-- bundled in js (rollup iife) -->
      <!-- <script src="_static/theme-vendors.js"></script> -->
      <script src="_static/theme.js" defer></script>
    
  <link rel="index" title="Index" href="genindex.html" />
  <link rel="search" title="Search" href="search.html" />
  <link rel="next" title="Developer Guidelines" href="developer.html" />
  <link rel="prev" title="Catalog collection schemas" href="catalogs.html" /> 
  </head>

  <body>
    <div id="app">
    <div class="theme-container" :class="pageClasses"><navbar @toggle-sidebar="toggleSidebar">
  <router-link to="index.html" class="home-link">
    
      <span class="site-name">Fritz Marshal</span>
    
  </router-link>

  <div class="links">
    <navlinks class="can-hide">



    </navlinks>
  </div>
</navbar>

      
      <div class="sidebar-mask" @click="toggleSidebar(false)">
      </div>
        <sidebar @toggle-sidebar="toggleSidebar">
          
          <navlinks>
            



            
          </navlinks><div id="searchbox" class="searchbox" role="search">
  <div class="caption"><span class="caption-text">Quick search</span>
    <div class="searchformwrapper">
      <form class="search" action="search.html" method="get">
        <input type="text" name="q" />
        <input type="submit" value="Search" />
        <input type="hidden" name="check_keywords" value="yes" />
        <input type="hidden" name="area" value="default" />
      </form>
    </div>
  </div>
</div><div class="sidebar-links" role="navigation" aria-label="main navigation">
  
    <div class="sidebar-group">
      <p class="caption">
        <span class="caption-text"><a href="index.html#fritz-marshal-manual">fritz marshal manual</a></span>
      </p>
      <ul class="current">
        
          <li class="toctree-l1 ">
            
              <a href="user_guide.html" class="reference internal ">Fritz User Guide</a>
            

            
          </li>

        
          <li class="toctree-l1 ">
            
              <a href="run_your_own.html" class="reference internal ">Run your own Fritz</a>
            

            
          </li>

        
          <li class="toctree-l1 ">
            
              <a href="api.html" class="reference internal ">API</a>
            

            
          </li>

        
          <li class="toctree-l1 ">
            
              <a href="catalogs.html" class="reference internal ">Catalog collection schemas</a>
            

            
          </li>

        
          <li class="toctree-l1 current">
            
              <a href="#" class="reference internal current">Technical implementation</a>
            

            
              <ul>
                
                  <li class="toctree-l2"><a href="#boom-and-skyportal" class="reference internal">BOOM and SkyPortal</a></li>
                
              </ul>
            
          </li>

        
          <li class="toctree-l1 ">
            
              <a href="developer.html" class="reference internal ">Developer Guidelines</a>
            

            
          </li>

        
          <li class="toctree-l1 ">
            
              <a href="license.html" class="reference internal ">License</a>
            

            
          </li>

        
      </ul>
    </div>
  
</div>
        </sidebar>

      <page>
          <div class="body-header" role="navigation" aria-label="navigation">
  
  <ul class="breadcrumbs">
    <li><a href="index.html">Docs</a> &raquo;</li>
    
    <li>Technical implementation</li>
  </ul>
  

  <ul class="page-nav">
  <li class="prev">
    <a href="catalogs.html"
       title="previous chapter">← Catalog collection schemas</a>
  </li>
  <li class="next">
    <a href="developer.html"
       title="next chapter">Developer Guidelines →</a>
  </li>
</ul>
  
</div>
<hr>
          <div class="content" role="main" v-pre>
            
  <section id="technical-implementation">
<h1>Technical implementation<a class="headerlink" href="#technical-implementation" title="Link to this heading">¶</a></h1>
<p>This is the TL;DR section of the docs.</p>
<p>The key characteristics of Fritz are efficiency, scalability, portability, and extensibility.
Fritz employs a modular architecture and
integrates and extends two major components: <a class="reference external" href="https://github.com/boom-astro/boom">BOOM</a>
acts as the alert processor and data archive, and <a class="reference external" href="https://github.com/skyportal/skyportal">SkyPortal</a>,
which handles the rest of the stack.
The schematic overview of our system is shown below:</p>
<p><img alt="img/fritz.png" src="_images/fritz.jpg" /></p>
<section id="boom-and-skyportal">
<h2>BOOM and SkyPortal<a class="headerlink" href="#boom-and-skyportal" title="Link to this heading">¶</a></h2>
<p>A non-relational (NoSQL) database <code class="docutils literal notranslate"><span class="pre">MongoDB</span></code> powers the data archive, the alert stream sink,
and the alert handling service. We base the choice of <code class="docutils literal notranslate"><span class="pre">MongoDB</span></code> as the workhorse on the following reasons:</p>
<ul class="simple">
<li><p>Individual alerts are stored as binary <code class="docutils literal notranslate"><span class="pre">JSON</span></code> (<code class="docutils literal notranslate"><span class="pre">BSON</span></code>) “documents” in “collections”,
a natural format for the <code class="docutils literal notranslate"><span class="pre">AVRO</span></code> alert packets.</p></li>
<li><p>Collections are implemented as B-trees, which guarantees <code class="docutils literal notranslate"><span class="pre">~log(N)</span></code> execution times for the standard
CRUD operations, where <code class="docutils literal notranslate"><span class="pre">N</span></code> is the number of documents in the collection.</p></li>
<li><p>Collections support multiple (compound) indexes and associated (in-memory) “covered” queries,
which will provide efficient access to the most-in-demand data.</p></li>
<li><p>Being a NoSQL database, <code class="docutils literal notranslate"><span class="pre">MongoDB</span></code> does not enforce any schema by default
meaning no downtime in case of an alert packet schema change.</p></li>
<li><p>Built-in <code class="docutils literal notranslate"><span class="pre">GeoJSON</span></code> support with 2D indexes on the sphere allowing efficient (potentially complicated) positional queries.</p></li>
<li><p>Built-in support for horizontal scaling through sharding.</p></li>
</ul>
<p>An HTTP API layer provides the interface to the backend, and is what SkyPortal talks to:
Fritz reaches it through SkyPortal’s broker API abstraction, so alert queries, filter
testing and photometry retrieval all go through one documented surface.</p>
<p>An alert brokering layer listens to <code class="docutils literal notranslate"><span class="pre">Kafka</span></code> alert streams and processes each packet,
which includes data preprocessing, execution of machine learning models,
catalog cross-matching, and ingestion into <code class="docutils literal notranslate"><span class="pre">MongoDB</span></code>.
It also executes user-defined filters based on the augmented alert data and posts the filtering results
to a <code class="docutils literal notranslate"><span class="pre">SkyPortal</span></code> instance, where they appear as Candidates on the relevant scanning page.</p>
<p><a class="reference external" href="https://skyportal.io"><code class="docutils literal notranslate"><span class="pre">SkyPortal</span></code></a>, similarly being an API-driven system, provides an interactive marshal
and a follow-up observation manager with flawless integration with both robotic and classical follow-up facilities.
It is designed with extensibility and scalability in mind: it uses an asynchronous <code class="docutils literal notranslate"><span class="pre">python</span></code> <code class="docutils literal notranslate"><span class="pre">tornado</span></code>-based backend,
and a <code class="docutils literal notranslate"><span class="pre">React</span></code>/<code class="docutils literal notranslate"><span class="pre">Redux</span></code>-powered frontend. It is containerized and can be orchestrated with <code class="docutils literal notranslate"><span class="pre">docker-compose</span></code> or <code class="docutils literal notranslate"><span class="pre">kubernetes</span></code>.
<code class="docutils literal notranslate"><span class="pre">SkyPortal</span></code> features real-time Slack-like messaging with notifications (via email or text);
fine-grained access control with ACLs and roles; rich visualization capabilities;
authentication via OAuth; and distributed computation via Dask.</p>
</section>
</section>


          </div>
          <div class="page-nav">
            <div class="inner"><ul class="page-nav">
  <li class="prev">
    <a href="catalogs.html"
       title="previous chapter">← Catalog collection schemas</a>
  </li>
  <li class="next">
    <a href="developer.html"
       title="next chapter">Developer Guidelines →</a>
  </li>
</ul><div class="footer" role="contentinfo">
      &#169; Copyright 2020, The Fritz Marshal authors.
    <br>
    Created using <a href="http://sphinx-doc.org/">Sphinx</a> 7.4.7 with <a href="https://github.com/schettino72/sphinx_press_theme">Press Theme</a> 0.9.1.
</div>
            </div>
          </div>
      </page>
    </div></div>
    
    
  </body>
</html>