casas-plotms-in-a-notebook.html

<!DOCTYPE html>
<html lang="en">
<link rel="stylesheet" type="text/css" href="https://keflavich.github.io/blog/theme/css/style.css" />
<link rel="icon" type="image/gif" href="https://keflavich.github.io/blog/theme/favicon8.ico">
<head>
    <base href="https://keflavich.github.io/blog">
        <title>Adam Ginsburg's blog</title>
        <meta charset="utf-8" />
        <link href="https://keflavich.github.io/blog/feeds/all.atom.xml" type="application/atom+xml" rel="alternate" title="Adam Ginsburg's blog Full Atom Feed" />
        <link href="https://keflavich.github.io/blog/feeds/{slug}.atom.xml" type="application/atom+xml" rel="alternate" title="Adam Ginsburg's blog Categories Atom Feed" />
<!-- Using MathJax, with the delimiters $ -->
<!-- Conflict with pygments for the .mo and .mi -->
<script type="text/x-mathjax-config">
    MathJax.Hub.Config({
    "HTML-CSS": {
    styles: {
    ".MathJax .mo, .MathJax .mi": {color: "inherit !important"}},
    'div.typeset': { 'text-align': 'left'}
    },
    tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']],processEscapes: true}
    });
    TeX: {
        extensions: ["AMSmath.js", "AMSsymbols.js"]
    }
    MathJax.Hub.Register.StartupHook("HTML-CSS Jax Ready",function () {
    var VARIANT = MathJax.OutputJax["HTML-CSS"].FONTDATA.VARIANT;
    VARIANT["normal"].fonts.unshift("MathJax_SansSerif");
    VARIANT["bold"].fonts.unshift("MathJax_SansSerif-bold");
    VARIANT["italic"].fonts.unshift("MathJax_SansSerif-italic");
    VARIANT["-tex-mathit"].fonts.unshift("MathJax_SansSerif-italic");
    });
    MathJax.Hub.Register.StartupHook("SVG Jax Ready",function () {
    var VARIANT = MathJax.OutputJax.SVG.FONTDATA.VARIANT;
    VARIANT["normal"].fonts.unshift("MathJax_SansSerif");
    VARIANT["bold"].fonts.unshift("MathJax_SansSerif-bold");
    VARIANT["italic"].fonts.unshift("MathJax_SansSerif-italic");
    VARIANT["-tex-mathit"].fonts.unshift("MathJax_SansSerif-italic");
    });
</script>

<script type="text/javascript"
src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML">
</script>
        
</head>

<body id="base" class="home">


    <header id="banner" class="body" >
        <div id="header" style='background-image: url("https://keflavich.github.io/blog/images/GC_4096sq_bolo.png"); background-position:left; min-heigt: 200px;  background-repeat: no-repeat; max-width: 80%;'>
            <h1 style="color: #C4C4C4;"><a class="header" href="https://keflavich.github.io/blog">Adam Ginsburg's blog <strong></strong></a></h1>

            <nav id="menu"><ul id="menulist">
                <li><a href="https://www.adamgginsburg.com">Homepage</a></li>
                <li><a href="/blog//index.html">Blog Index</a></li>
                <li><a href="/blog//blog/category/bgps.html">BGPS Blog</a></li>
                <li><a href="/blog//blog/category/publications.html">Publications</a></li>
                <li><a href="/blog//blog/archives.html">Archives</a></li>
                <li><a href="/blog//blog/tags.html">Tags</a></li>
            </ul></nav><!-- /#menu -->
        </div>
    </header><!-- /#banner -->

  <div id="sidebar">
    <ul>
      <li>
        <h3 id='recent_header'>Recent Posts</h3>
        <ul>
              <li class="post">
                  2025/03/15
                  <br>
                  <a href="https://keflavich.github.io/blog/pushing-to-a-pull-request.html">Pushing to a pull request</a>
              </li>
              <li class="post">
                  2023/08/11
                  <br>
                  <a href="https://keflavich.github.io/blog/editing-metadata-in-measurement-sets.html">Editing metadata in measurement sets</a>
              </li>
              <li class="post">
                  2022/05/06
                  <br>
                  <a href="https://keflavich.github.io/blog/casa-mpi-debugging-contd.html">CASA MPI debugging cont'd</a>
              </li>
              <li class="post">
                  2022/05/01
                  <br>
                  <a href="https://keflavich.github.io/blog/co2-monitoring-at-conferences-update-in-2022.html">CO2 Monitoring at Conferences: Update in 2022</a>
              </li>
              <li class="post">
                  2022/04/14
                  <br>
                  <a href="https://keflavich.github.io/blog/alma-cycle-9-corrupted-zip-fix.html">ALMA Cycle 9 corrupted zip fix</a>
              </li>
        </ul>
      </li>
    
    </ul>
  </div><!-- end #sidebar -->

  <div id="content">
<section id="content" class="body">
  <header>
    <h2 class="entry-title">
      <a href="https://keflavich.github.io/blog/casas-plotms-in-a-notebook.html" rel="bookmark"
         title="Permalink to casa's plotms in a notebook">casa's plotms in a notebook</a></h2>
 
  </header>
  <footer class="post-info">
    <abbr class="published" title="2021-06-14T12:30:00-06:00">
      2021/06/14
    </abbr>
    <address class="vcard author">
      By <a class="url fn" href="https://keflavich.github.io/blog/author/adam-ginsburg-adamgginsburggmailcom.html">Adam Ginsburg (Adam.G.Ginsburg@gmail.com)</a>
    </address>
  </footer><!-- /.post-info -->
  <div class="entry-content">
    <p>CASA's plotms requires an X11 frontend to work properly, so you either need to run it with xvfb or
with X.</p>
<p>I want to just make UV data plots in jupyter notebooks.</p>
<p>This is harder than it sounds, particularly for concatenated data sets.</p>
<p>First step is that you need to assemble the metadata:</p>
<div class="highlight"><pre><span></span><span class="n">msmd</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">scvis</span><span class="p">)</span>
<span class="n">scsum</span> <span class="o">=</span> <span class="n">msmd</span><span class="o">.</span><span class="n">summary</span><span class="p">()</span>
<span class="n">msmd</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
</pre></div>
<p>The resulting <tt class="docutils literal">scsum</tt> is a nested dictionary containing a ton of information.
We need only some of it.</p>
<p>To get out some flattened information, such as the &quot;data description IDs&quot; associated
with data that are actually in my concatenated MS (in my case, 8 of 11 observation blocks
in the metadata are not present in the data), this tool gets and flattens the resulting
information:</p>
<div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">walk_summary</span><span class="p">(</span><span class="n">summary</span><span class="p">,</span> <span class="n">key</span><span class="p">):</span>
    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">summary</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">summary</span><span class="p">:</span>
                <span class="k">return</span> <span class="n">summary</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">result</span> <span class="o">=</span> <span class="p">[</span><span class="n">walk_summary</span><span class="p">(</span><span class="n">summary</span><span class="p">[</span><span class="n">kk</span><span class="p">],</span> <span class="n">key</span><span class="p">)</span> <span class="k">for</span> <span class="n">kk</span> <span class="ow">in</span> <span class="n">summary</span><span class="p">]</span>
                <span class="k">try</span><span class="p">:</span>
                    <span class="n">result</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">unique</span><span class="p">([</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">result</span> <span class="k">if</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">])</span>
                    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">result</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
                        <span class="k">return</span> <span class="n">result</span>
                <span class="k">except</span><span class="p">:</span>
                    <span class="k">return</span> <span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">result</span> <span class="k">if</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">]</span>
        <span class="k">except</span> <span class="p">(</span><span class="ne">KeyError</span><span class="p">,</span><span class="ne">TypeError</span><span class="p">):</span>
            <span class="k">return</span>
</pre></div>
<p>Then we grab the usable observation and data description IDs:</p>
<div class="highlight"><pre><span></span><span class="n">obsids</span> <span class="o">=</span> <span class="p">[</span><span class="n">key</span> <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">scsum</span> <span class="k">if</span> <span class="s1">&#39;observationID&#39;</span> <span class="ow">in</span> <span class="n">key</span> <span class="ow">and</span> <span class="n">scsum</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">!=</span> <span class="p">{}]</span>
<span class="n">ddids</span> <span class="o">=</span> <span class="n">walk_summary</span><span class="p">(</span><span class="n">scsum</span><span class="p">,</span> <span class="s1">&#39;data description IDs&#39;</span><span class="p">)</span>
</pre></div>
<p>Now we have to load the actual data.  I needed to create new versions of the <tt class="docutils literal">mstool</tt> each time
because I was getting weird errors where <tt class="docutils literal">ms.close()</tt> followed by <tt class="docutils literal">ms.open(fn)</tt> was not properly
resetting, which resulted in <tt class="docutils literal">ms.getdata</tt> returning the same data for two different MSes - BAD!</p>
<div class="highlight"><pre><span></span><span class="n">ms6</span> <span class="o">=</span> <span class="n">mstool</span><span class="p">()</span>
<span class="n">ms6</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">scvis</span><span class="p">)</span>
<span class="n">scdata_all</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">for</span> <span class="n">ddid</span> <span class="ow">in</span> <span class="n">ddids</span><span class="p">:</span>
    <span class="n">ms6</span><span class="o">.</span><span class="n">selectinit</span><span class="p">(</span><span class="n">ddid</span><span class="p">,</span> <span class="n">reset</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
    <span class="n">ms6</span><span class="o">.</span><span class="n">selectinit</span><span class="p">(</span><span class="n">ddid</span><span class="p">)</span>
    <span class="n">scdata_all</span><span class="p">[</span><span class="n">ddid</span><span class="p">]</span> <span class="o">=</span> <span class="n">ms6</span><span class="o">.</span><span class="n">getdata</span><span class="p">([</span><span class="s1">&#39;amplitude&#39;</span><span class="p">,</span> <span class="s1">&#39;weight&#39;</span><span class="p">,</span> <span class="s1">&#39;uvdist&#39;</span><span class="p">,</span> <span class="s1">&#39;axis_info&#39;</span><span class="p">,</span> <span class="s1">&#39;flag&#39;</span><span class="p">,</span> <span class="s1">&#39;corrected_amplitude&#39;</span><span class="p">],</span> <span class="n">ifraxis</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">ms6</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
</pre></div>
<p>Then plotting is a hassle, but at least is not insane - it's just normal array manipulation:</p>
<div class="highlight"><pre><span></span><span class="n">pl</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">12</span><span class="p">,</span><span class="mi">12</span><span class="p">))</span>
<span class="n">colors</span> <span class="o">=</span> <span class="n">itertools</span><span class="o">.</span><span class="n">cycle</span><span class="p">(</span><span class="n">pl</span><span class="o">.</span><span class="n">rcParams</span><span class="p">[</span><span class="s2">&quot;axes.prop_cycle&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">by_key</span><span class="p">()[</span><span class="s2">&quot;color&quot;</span><span class="p">])</span>
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">scdata_all</span><span class="p">:</span> <span class="c1"># the keys are SPW numbers from the multi-MS file</span>
    <span class="n">weight</span> <span class="o">=</span> <span class="n">scdata_all</span><span class="p">[</span><span class="n">key</span><span class="p">][</span><span class="s1">&#39;weight&#39;</span><span class="p">]</span>
    <span class="n">uvd</span> <span class="o">=</span> <span class="n">scdata_all</span><span class="p">[</span><span class="n">key</span><span class="p">][</span><span class="s1">&#39;uvdist&#39;</span><span class="p">]</span>
    <span class="n">pl</span><span class="o">.</span><span class="n">semilogy</span><span class="p">(</span><span class="n">uvd</span><span class="o">.</span><span class="n">ravel</span><span class="p">(),</span> <span class="n">weight</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">weight</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">uvd</span><span class="o">.</span><span class="n">size</span><span class="p">)</span><span class="o">.</span><span class="n">T</span><span class="p">,</span> <span class="s1">&#39;,&#39;</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="nb">next</span><span class="p">(</span><span class="n">colors</span><span class="p">),</span> <span class="n">label</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)</span>
<span class="n">pl</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="s1">&#39;best&#39;</span><span class="p">)</span>
<span class="n">pl</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s2">&quot;UV Distance (m)&quot;</span><span class="p">)</span>
<span class="n">pl</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s2">&quot;Weight&quot;</span><span class="p">)</span>
</pre></div>
<p>That is basically equivalent to <tt class="docutils literal">plotms(vis, <span class="pre">xaxis='uvdist',</span> <span class="pre">yaxis='weight',</span> <span class="pre">coloraxis='spw')</span></tt>.  It's just slightly more flexible, since
you can also make waterfall plots, etc.</p>
<p>Selecting along various axes, like antenna, gets tricky.  You'd rather not go back to <tt class="docutils literal">ms.getdata</tt> because it is _slow_.</p>
<p>The &quot;identifying baselines&quot; step that you can do in plotms can be achieved through normal boolean array selection, with a result that's somewhat easier to read than CASA's logger:</p>
<div class="highlight"><pre><span></span><span class="n">highwt</span> <span class="o">=</span> <span class="p">(</span><span class="n">scdata_all</span><span class="p">[</span><span class="mi">56</span><span class="p">][</span><span class="s1">&#39;weight&#39;</span><span class="p">]</span> <span class="o">&gt;</span> <span class="mf">1e5</span><span class="p">)</span>
<span class="n">scdata_all</span><span class="p">[</span><span class="mi">56</span><span class="p">][</span><span class="s1">&#39;axis_info&#39;</span><span class="p">][</span><span class="s1">&#39;ifr_axis&#39;</span><span class="p">][</span><span class="s1">&#39;ifr_name&#39;</span><span class="p">][</span><span class="n">highwt</span><span class="o">.</span><span class="n">any</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span><span class="mi">2</span><span class="p">))]</span>
<span class="c1"># array([&#39;DA41-DV12&#39;, &#39;DA42-DV12&#39;, &#39;DA43-DV12&#39;, &#39;DA44-DV12&#39;, &#39;DA45-DV12&#39;,</span>
<span class="c1">#  &#39;DA46-DV12&#39;, &#39;DA47-DV12&#39;, &#39;DA48-DV12&#39;, &#39;DA49-DV12&#39;, &#39;DA50-DV12&#39;,</span>
<span class="c1">#  &#39;DA51-DV12&#39;, &#39;DA52-DV12&#39;, &#39;DA53-DV12&#39;, &#39;DA54-DV12&#39;, &#39;DA55-DV12&#39;,</span>
<span class="c1">#  &#39;DA56-DV12&#39;, &#39;DA57-DV12&#39;, &#39;DA58-DV12&#39;, &#39;DA60-DV12&#39;, &#39;DA61-DV12&#39;,</span>
<span class="c1">#  &#39;DA62-DV12&#39;, &#39;DA63-DV12&#39;, &#39;DA64-DV12&#39;, &#39;DA65-DV12&#39;, &#39;DV01-DV12&#39;,</span>
<span class="c1">#  &#39;DV02-DV12&#39;, &#39;DV03-DV12&#39;, &#39;DV04-DV12&#39;, &#39;DV05-DV12&#39;, &#39;DV06-DV12&#39;,</span>
<span class="c1">#  &#39;DV07-DV12&#39;, &#39;DV08-DV12&#39;, &#39;DV09-DV12&#39;, &#39;DV10-DV12&#39;, &#39;DV11-DV12&#39;,</span>
<span class="c1">#  &#39;DV12-DV14&#39;, &#39;DV12-DV16&#39;, &#39;DV12-DV17&#39;, &#39;DV12-DV19&#39;, &#39;DV12-DV20&#39;,</span>
<span class="c1">#  &#39;DV12-DV21&#39;, &#39;DV12-DV22&#39;, &#39;DV12-DV23&#39;, &#39;DV12-DV24&#39;, &#39;DV12-DV25&#39;],</span>
<span class="c1"># dtype=&#39;&lt;U16&#39;)</span>
</pre></div>
<p>in this case, the output shows that DV12 is clearly the antenna that's overweighted.</p>
<p>Handling flags is tricky.  The 'weight' array has shape <tt class="docutils literal">[2,nbaseline,ntime]</tt>, but I don't know if that 2 refers to frequency
or polarization, since I have two of each and there is no information about this in the <tt class="docutils literal">axis_info</tt> dictionary.  So... I'm
doing the conservative thing and ignoring any row of frequency or polarization if _any_ of the data are flagged.</p>
<div class="highlight"><pre><span></span><span class="n">pl</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">12</span><span class="p">,</span><span class="mi">12</span><span class="p">))</span>
<span class="n">colors</span> <span class="o">=</span> <span class="n">itertools</span><span class="o">.</span><span class="n">cycle</span><span class="p">(</span><span class="n">pl</span><span class="o">.</span><span class="n">rcParams</span><span class="p">[</span><span class="s2">&quot;axes.prop_cycle&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">by_key</span><span class="p">()[</span><span class="s2">&quot;color&quot;</span><span class="p">])</span>
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">scdata_all</span><span class="p">:</span>
    <span class="n">weight</span> <span class="o">=</span> <span class="n">scdata_all</span><span class="p">[</span><span class="n">key</span><span class="p">][</span><span class="s1">&#39;weight&#39;</span><span class="p">]</span>
    <span class="n">uvd</span> <span class="o">=</span> <span class="n">scdata_all</span><span class="p">[</span><span class="n">key</span><span class="p">][</span><span class="s1">&#39;uvdist&#39;</span><span class="p">]</span>
    <span class="n">flag</span> <span class="o">=</span> <span class="n">scdata_all</span><span class="p">[</span><span class="n">key</span><span class="p">][</span><span class="s1">&#39;flag&#39;</span><span class="p">]</span>
    <span class="n">okflag</span> <span class="o">=</span> <span class="o">~</span><span class="n">np</span><span class="o">.</span><span class="n">any</span><span class="p">(</span><span class="n">flag</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span><span class="mi">1</span><span class="p">))</span>
    <span class="n">pl</span><span class="o">.</span><span class="n">semilogy</span><span class="p">(</span><span class="n">uvd</span><span class="p">[</span><span class="n">okflag</span><span class="p">],</span> <span class="n">weight</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">weight</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">uvd</span><span class="o">.</span><span class="n">size</span><span class="p">)</span><span class="o">.</span><span class="n">T</span><span class="p">[</span><span class="n">okflag</span><span class="o">.</span><span class="n">ravel</span><span class="p">(),</span> <span class="p">:],</span> <span class="s1">&#39;,&#39;</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="nb">next</span><span class="p">(</span><span class="n">colors</span><span class="p">),</span> <span class="n">label</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)</span>
<span class="n">pl</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="s1">&#39;best&#39;</span><span class="p">)</span>
<span class="n">_</span><span class="o">=</span><span class="n">pl</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s2">&quot;UV Distance (m)&quot;</span><span class="p">)</span>
<span class="n">_</span><span class="o">=</span><span class="n">pl</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s2">&quot;Weight&quot;</span><span class="p">)</span>
</pre></div>
<p>And, indeed, doing this revealed that the super-high-weight antenna was already flagged out.</p>
<p>This has been a demo of how to do some stuff with CASA to replicate plotms</p>
<div class="section" id="errors">
<h2>Errors</h2>
<p>Along the way, I hit a ton of errors I didn't understand, so I need a record of what they mean.</p>
<div class="section" id="data-shape-varies-selecting-first-data-desc-id-only">
<h3>&quot;Data shape varies, selecting first data desc id only&quot;</h3>
<p>This is the worst.  It means that you have a multi-ms file (mms) that has been produced by <tt class="docutils literal">concat</tt>.  It is therefore doing something <em>explicitly wrong</em> and returning only the first data description ID.</p>
<p>Data description IDs are super counterintuitive.  They refer, afaict, to SPW numbers.  If you have an MMS with unmerged SPWs (same frequency, but you didn't merge them because of tolerances or something),
each SPW will correspond to a single observation ID.</p>
<p>The solution is to explicitly <tt class="docutils literal">ms.selectinit</tt>, i.e.:</p>
<div class="highlight"><pre><span></span><span class="n">ms</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">vis</span><span class="p">)</span>
<span class="n">ms</span><span class="o">.</span><span class="n">selectinit</span><span class="p">(</span><span class="n">datadescid</span><span class="p">)</span> <span class="c1"># note: &quot;reset=True&quot; appears to _override_ the selection</span>
<span class="n">ms</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="o">...</span><span class="p">)</span> <span class="c1"># use this as needed</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">ms</span><span class="o">.</span><span class="n">getdata</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="n">ms</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
</pre></div>
<p>but <tt class="docutils literal">datadescid</tt> is _not_ the observation ID.  It's just... some number you need to get from the metadata.</p>
</div>
<div class="section" id="mismatched-data-shapes-when-comparing-two-mses">
<h3>Mismatched data shapes when comparing two MSes</h3>
<p>I got into this whole thing because I wanted to compare the data in two MSes to determine where we went wrong (we did go wrong).</p>
<p>Unfortunately, somewhere along the way, the autocorrelations got dropped.  I don't know how it happened because I never did it
(but I wish I had earlier), but that means that now the data shapes are mismatched.</p>
<p>The solution was to use <tt class="docutils literal">ms.select</tt> and explicitly downselect to the baselines from MS #1 in MS #2 using MS #1's <tt class="docutils literal">axis_info</tt>: <tt class="docutils literal">{'ifr_number': <span class="pre">scdata['axis_info']['ifr_axis']['ifr_number']}</span></tt>:</p>
<div class="highlight"><pre><span></span><span class="n">ms3</span> <span class="o">=</span> <span class="n">mstool</span><span class="p">()</span>
<span class="n">ms3</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">newvis</span><span class="p">)</span>
<span class="n">ms3</span><span class="o">.</span><span class="n">selectinit</span><span class="p">(</span><span class="mi">80</span><span class="p">)</span> <span class="c1"># 80? what the hell kind of datadescid is that?!</span>
<span class="n">ms3</span><span class="o">.</span><span class="n">select</span><span class="p">({</span><span class="s1">&#39;ifr_number&#39;</span><span class="p">:</span> <span class="n">scdata</span><span class="p">[</span><span class="s1">&#39;axis_info&#39;</span><span class="p">][</span><span class="s1">&#39;ifr_axis&#39;</span><span class="p">][</span><span class="s1">&#39;ifr_number&#39;</span><span class="p">]})</span>
<span class="n">newdata</span> <span class="o">=</span> <span class="n">ms3</span><span class="o">.</span><span class="n">getdata</span><span class="p">([</span><span class="s1">&#39;amplitude&#39;</span><span class="p">,</span> <span class="s1">&#39;uvdist&#39;</span><span class="p">,</span> <span class="s1">&#39;axis_info&#39;</span><span class="p">,</span> <span class="s1">&#39;corrected_amplitude&#39;</span><span class="p">],</span> <span class="n">ifraxis</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">ms3</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="n">newdata</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
</pre></div>
<p>That downselected the baselines to match the baselines present in <tt class="docutils literal">scdata</tt>, which was MS 1.</p>
</div>
<div class="section" id="the-wrong-data-are-showing-up-when-i-open-a-new-ms">
<h3>The wrong data are showing up when I open a new MS</h3>
<p>I was comparing several MSes, and sometimes the data looked the same for MSes I was <em>dead certain</em> were different.
I was right, and the problem was apparently reusing the ms tool.</p>
<p>I didn't show the import statement above, but it was this:</p>
<div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">casatools</span> <span class="kn">import</span> <span class="n">ms</span><span class="p">,</span> <span class="n">msmetadata</span>
<span class="n">mstool</span> <span class="o">=</span> <span class="n">ms</span>
<span class="n">ms</span> <span class="o">=</span> <span class="n">ms</span><span class="p">()</span>
<span class="n">msmd</span> <span class="o">=</span> <span class="n">msmetadata</span><span class="p">()</span>
</pre></div>
<p>If I tried to reuse <tt class="docutils literal">ms</tt> as normal, like so:</p>
<div class="highlight"><pre><span></span><span class="n">ms</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">vis1</span><span class="p">)</span>
<span class="n">ms</span><span class="o">.</span><span class="n">selectinit</span><span class="p">(</span><span class="mi">80</span><span class="p">)</span>
<span class="n">data1</span> <span class="o">=</span> <span class="n">ms</span><span class="o">.</span><span class="n">getdata</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="n">ms</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>

<span class="n">ms</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">vis2</span><span class="p">)</span>
<span class="n">ms</span><span class="o">.</span><span class="n">selectinit</span><span class="p">(</span><span class="mi">80</span><span class="p">)</span>
<span class="n">data2</span> <span class="o">=</span> <span class="n">ms</span><span class="o">.</span><span class="n">getdata</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="n">ms</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
</pre></div>
<p>then I can't say exactly what <tt class="docutils literal">data1</tt> and <tt class="docutils literal">data2</tt> are, but they weren't the same.</p>
<p>So, instead, I created a new <tt class="docutils literal">mstool</tt> instance each time:</p>
<div class="highlight"><pre><span></span><span class="n">ms</span> <span class="o">=</span> <span class="n">mstool</span><span class="p">()</span>
<span class="n">ms</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">vis1</span><span class="p">)</span>
<span class="n">ms</span><span class="o">.</span><span class="n">selectinit</span><span class="p">(</span><span class="mi">80</span><span class="p">)</span>
<span class="n">data1</span> <span class="o">=</span> <span class="n">ms</span><span class="o">.</span><span class="n">getdata</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="n">ms</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>

<span class="n">ms</span> <span class="o">=</span> <span class="n">mstool</span><span class="p">()</span>
<span class="n">ms</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">vis2</span><span class="p">)</span>
<span class="n">ms</span><span class="o">.</span><span class="n">selectinit</span><span class="p">(</span><span class="mi">80</span><span class="p">)</span>
<span class="n">data2</span> <span class="o">=</span> <span class="n">ms</span><span class="o">.</span><span class="n">getdata</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="n">ms</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
</pre></div>
</div>
</div>
<div class="section" id="put-all-together">
<h2>Put all together...</h2>
<p>Here's some code to loop over a list of MSes and plot both amp and weight vs. uvdist for the field W43-MM3.</p>
<p>Of course this crashed immediately because it exceeded the available memory.</p>
<div class="highlight"><pre><span></span><span class="k">for</span> <span class="n">vis</span> <span class="ow">in</span> <span class="n">mses</span><span class="p">:</span>
    <span class="n">msmd</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">vis</span><span class="p">)</span>
    <span class="n">vissum</span> <span class="o">=</span> <span class="n">msmd</span><span class="o">.</span><span class="n">summary</span><span class="p">()</span>
    <span class="n">msmd</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>

    <span class="n">vobsids</span> <span class="o">=</span> <span class="p">[</span><span class="n">key</span> <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">vissum</span> <span class="k">if</span> <span class="s1">&#39;observationID&#39;</span> <span class="ow">in</span> <span class="n">key</span> <span class="ow">and</span> <span class="n">vissum</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">!=</span> <span class="p">{}]</span>
    <span class="n">vddids</span> <span class="o">=</span> <span class="n">walk_summary</span><span class="p">(</span><span class="n">vissum</span><span class="p">,</span> <span class="s1">&#39;data description IDs&#39;</span><span class="p">)</span>
    <span class="n">fids</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">vissum</span><span class="p">[</span><span class="s1">&#39;fields&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;W43-MM3&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;obsids: </span><span class="si">{</span><span class="n">vobsids</span><span class="si">}</span><span class="s2">, ddids: </span><span class="si">{</span><span class="n">vddids</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>

    <span class="n">ms</span> <span class="o">=</span> <span class="n">mstool</span><span class="p">()</span>
    <span class="n">ms</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">vis</span><span class="p">)</span>
    <span class="n">msdata_all</span> <span class="o">=</span> <span class="p">{}</span>
    <span class="k">for</span> <span class="n">ddid</span> <span class="ow">in</span> <span class="n">vddids</span><span class="p">:</span>
        <span class="n">ms</span><span class="o">.</span><span class="n">selectinit</span><span class="p">(</span><span class="n">ddid</span><span class="p">,</span> <span class="n">reset</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
        <span class="n">ms</span><span class="o">.</span><span class="n">selectinit</span><span class="p">(</span><span class="n">ddid</span><span class="p">)</span>
        <span class="n">ms</span><span class="o">.</span><span class="n">select</span><span class="p">({</span><span class="s1">&#39;field_id&#39;</span><span class="p">:</span> <span class="n">fids</span><span class="p">})</span>
        <span class="n">msdata_all</span><span class="p">[</span><span class="n">ddid</span><span class="p">]</span> <span class="o">=</span> <span class="n">ms</span><span class="o">.</span><span class="n">getdata</span><span class="p">([</span><span class="s1">&#39;amplitude&#39;</span><span class="p">,</span> <span class="s1">&#39;weight&#39;</span><span class="p">,</span> <span class="s1">&#39;uvdist&#39;</span><span class="p">,</span> <span class="s1">&#39;axis_info&#39;</span><span class="p">,</span> <span class="s1">&#39;flag&#39;</span><span class="p">,</span> <span class="s1">&#39;corrected_amplitude&#39;</span><span class="p">],</span> <span class="n">ifraxis</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
    <span class="n">ms</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>

    <span class="n">colors</span> <span class="o">=</span> <span class="n">itertools</span><span class="o">.</span><span class="n">cycle</span><span class="p">(</span><span class="n">pl</span><span class="o">.</span><span class="n">rcParams</span><span class="p">[</span><span class="s2">&quot;axes.prop_cycle&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">by_key</span><span class="p">()[</span><span class="s2">&quot;color&quot;</span><span class="p">])</span>
    <span class="n">pl</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">12</span><span class="p">,</span><span class="mi">12</span><span class="p">))</span>

    <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">msdata_all</span><span class="p">:</span>
        <span class="n">weight</span> <span class="o">=</span> <span class="n">msdata_all</span><span class="p">[</span><span class="n">key</span><span class="p">][</span><span class="s1">&#39;weight&#39;</span><span class="p">]</span>
        <span class="n">uvd</span> <span class="o">=</span> <span class="n">msdata_all</span><span class="p">[</span><span class="n">key</span><span class="p">][</span><span class="s1">&#39;uvdist&#39;</span><span class="p">]</span>
        <span class="n">flag</span> <span class="o">=</span> <span class="n">msdata_all</span><span class="p">[</span><span class="n">key</span><span class="p">][</span><span class="s1">&#39;flag&#39;</span><span class="p">]</span>
        <span class="n">okflag</span> <span class="o">=</span> <span class="o">~</span><span class="n">np</span><span class="o">.</span><span class="n">any</span><span class="p">(</span><span class="n">flag</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span><span class="mi">1</span><span class="p">))</span>
        <span class="n">pl</span><span class="o">.</span><span class="n">semilogy</span><span class="p">(</span><span class="n">uvd</span><span class="p">[</span><span class="n">okflag</span><span class="p">],</span> <span class="n">weight</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">weight</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">uvd</span><span class="o">.</span><span class="n">size</span><span class="p">)</span><span class="o">.</span><span class="n">T</span><span class="p">[</span><span class="n">okflag</span><span class="o">.</span><span class="n">ravel</span><span class="p">(),</span> <span class="p">:],</span> <span class="s1">&#39;,&#39;</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="nb">next</span><span class="p">(</span><span class="n">colors</span><span class="p">),</span> <span class="n">label</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)</span>
    <span class="n">pl</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="s1">&#39;best&#39;</span><span class="p">)</span>
    <span class="n">pl</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="n">vis</span><span class="p">)</span>
    <span class="n">_</span><span class="o">=</span><span class="n">pl</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s2">&quot;UV Distance (m)&quot;</span><span class="p">)</span>
    <span class="n">_</span><span class="o">=</span><span class="n">pl</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s2">&quot;Weight&quot;</span><span class="p">)</span>

    <span class="n">colors</span> <span class="o">=</span> <span class="n">itertools</span><span class="o">.</span><span class="n">cycle</span><span class="p">(</span><span class="n">pl</span><span class="o">.</span><span class="n">rcParams</span><span class="p">[</span><span class="s2">&quot;axes.prop_cycle&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">by_key</span><span class="p">()[</span><span class="s2">&quot;color&quot;</span><span class="p">])</span>
    <span class="n">pl</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">12</span><span class="p">,</span><span class="mi">12</span><span class="p">))</span>

    <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">msdata_all</span><span class="p">:</span>
        <span class="n">amp</span> <span class="o">=</span> <span class="n">msdata_all</span><span class="p">[</span><span class="n">key</span><span class="p">][</span><span class="s1">&#39;amplitude&#39;</span><span class="p">]</span>
        <span class="n">uvd</span> <span class="o">=</span> <span class="n">msdata_all</span><span class="p">[</span><span class="n">key</span><span class="p">][</span><span class="s1">&#39;uvdist&#39;</span><span class="p">]</span>
        <span class="n">flag</span> <span class="o">=</span> <span class="n">msdata_all</span><span class="p">[</span><span class="n">key</span><span class="p">][</span><span class="s1">&#39;flag&#39;</span><span class="p">]</span>
        <span class="n">okflag</span> <span class="o">=</span> <span class="o">~</span><span class="n">np</span><span class="o">.</span><span class="n">any</span><span class="p">(</span><span class="n">flag</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span><span class="mi">1</span><span class="p">))</span>
        <span class="n">pl</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">uvd</span><span class="p">[</span><span class="n">okflag</span><span class="p">],</span> <span class="n">amp</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">amp</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">*</span><span class="n">amp</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">uvd</span><span class="o">.</span><span class="n">size</span><span class="p">)</span><span class="o">.</span><span class="n">T</span><span class="p">[</span><span class="n">okflag</span><span class="o">.</span><span class="n">ravel</span><span class="p">(),</span> <span class="p">:],</span> <span class="s1">&#39;,&#39;</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="nb">next</span><span class="p">(</span><span class="n">colors</span><span class="p">),</span> <span class="n">label</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)</span>
    <span class="n">pl</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="s1">&#39;best&#39;</span><span class="p">)</span>
    <span class="n">pl</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="n">vis</span><span class="p">)</span>
    <span class="n">_</span><span class="o">=</span><span class="n">pl</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s2">&quot;UV Distance (m)&quot;</span><span class="p">)</span>
    <span class="n">_</span><span class="o">=</span><span class="n">pl</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s2">&quot;Amplitude&quot;</span><span class="p">)</span>
</pre></div>
</div>

  </div><!-- /.entry-content -->
</section>

  </div>

    <script type="text/javascript">
    var _gaq = _gaq || [];
    _gaq.push(['_setAccount', 'UA-37306139-1']);
    _gaq.push(['_trackPageview']);
    (function() {
        var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
        ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
        var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
    })();

    (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
    (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
    m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
    })(window,document,'script','//www.google-analytics.com/analytics.js','ga');

    ga('create', 'UA-37306139-1');
    ga('send', 'pageview');
</script>
<!-- Start of StatCounter Code for Default Guide -->
<script type="text/javascript">
var sc_project=9280906; 
var sc_invisible=1; 
var sc_security="7107d360"; 
var scJsHost = (("https:" == document.location.protocol) ?
"https://secure." : "http://www.");
document.write("<sc"+"ript type='text/javascript' src='" +
scJsHost+
"statcounter.com/counter/counter.js'></"+"script>");
</script>
<!--<noscript><div class="statcounter"><a title="hit counter"
href="http://statcounter.com/free-hit-counter/"
target="_blank"><img class="statcounter"
src="http://c.statcounter.com/9280906/0/7107d360/1/"
alt="hit counter"></a></div></noscript>-->
<!-- End of StatCounter Code for Default Guide -->
<script type="text/javascript">
  var disqus_shortname = 'adamginsburgsblog';
  var disqus_identifier = '/casas-plotms-in-a-notebook.html';
  var disqus_url = 'https://keflavich.github.io/blog/casas-plotms-in-a-notebook.html';
  var disqus_title = 'casa's plotms in a notebook';
  (function() {
        var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
        dsq.src = 'https://' + disqus_shortname + '.disqus.com/embed.js';
        (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
   })();
</script>
</body>
</html>