Experiment with Frequency MOC in MOCCli

CHANGELOG.md

@@ -1,5 +1,14 @@
 # `moc` Change Log
 
+## 0.9.0-alpha
+
+Released 2022-06-17
+
+### Added
+
+* Add Frequency MOCs
+
+
 ## 0.8.0
 
 Released 2022-04-13

Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "moc"
-version = "0.8.0"
+version = "0.9.0-alpha"
 authors = [
   "F.-X. Pineau <[email protected]>",
   "Matthieu Baumann <[email protected]>"

crates/cli/CHANGELOG.md

@@ -1,5 +1,13 @@
 # `moc-cli` Change Log
 
+## 0.5.0-alpha
+
+Realeased 2022-06-17
+
+* Add support for frequency MOCs
+* Add 'hprint' (human print) command for time and frequency
+
+
 ## 0.4.0
 
 Realeased 2022-04-13

crates/cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "moc-cli"
-version = "0.4.0"
+version = "0.5.0-alpha"
 authors = ["F.-X. Pineau <[email protected]>"]
 description = """
 moc is a command-line to create and manipulate HEALPix

crates/cli/README.md

@@ -178,6 +178,30 @@ moc info my_res.fits
 moc from cone 11 0.0 +0.0 0.1 ascii --fold 50 my_cone.ascii
 moc convert -t smoc my_cone.ascii fits -f my_cone.fits
 moc from ring 10 13.158329 -72.80028  5.64323 10.0 ascii --fold 80
+
+## Frequency MOCs
+echo "0/0 8/" | moc hprint --type fmoc --format ascii -
+echo "0/0-1 8/" | moc convert --type smoc --format ascii - fits moc.d8.allfreq.fits
+moc info moc.d8.allfreq.fits
+moc hprint moc.d8.allfreq.fits
+
+for f in {0..63}; do echo "5/$f" | moc hprint --type fmoc --format ascii - --no-header; done
+for f in {0..511}; do echo "8/$f" | moc hprint --type fmoc --format ascii - --no-header; done
+for f in {0..1023}; do echo "9/$f" | moc hprint --type fmoc --format ascii - --no-header; done
+
+for f in {0..8191}; do echo "12/$f" | moc hprint --type fmoc --format ascii - --no-header; done
+for f in {0..65536}; do echo "15/$f" | moc hprint --type fmoc --format ascii - --no-header; done
+
+echo "0.125
+0.2569
+0.1478
+0.9985
+1.5983
+20.256
+3500" | \
+moc from freqval 9 - ascii | \
+moc hprint --type fmoc --format ascii -
+
 ```
 
 Building a MOC from the [Hipparcos](https://vizier.u-strasbg.fr/viz-bin/VizieR-3?-source=I/239/hip_main&-out.max=50&-out.form=HTML%20Table&-out.add=_r&-out.add=_RAJ,_DEJ&-sort=_r&-oc.form=sexa)
@@ -222,6 +246,14 @@ MOC type: SPACE
 MOC index type: u64
 MOC depth: 12
 MOC coverage:   3.962597748 %
+
+# Count the number of disjoint regions from the 12210 FOVs
+> time moc op split --count /home/pineau/Eclipse/ARCHES/scriptsNoSync/xmm.moc.fits ascii
+4414
+
+real	0m2,617s
+user	0m2,604s
+sys	0m0,005s
 ```
 
 ![XMM MOC allsky](img/xmmmoc.png)
@@ -319,6 +351,8 @@ time moc filter position SMOC_GLIMPSE_u32.fits kids_dr2.csv --has-header --lon R
 
 ## To-do list
 
+* [ ] MOC from cells (simpler than MOC from pos)
+* [ ] MOC from vcells with a simple constraint? 
 * [ ] Contact [gnuastro](https://www.gnu.org/software/gnuastro) ?
 * [ ] Add filter on ST-MOCs
 * [ ] Add ST-MOC 'intersection' and 'folds' in streaming mode (for a low memory footprint)

crates/cli/src/constants.rs

@@ -3,23 +3,27 @@ use std::error::Error;
 
 use structopt::StructOpt;
 
-use moclib::qty::{MocQty, Hpx, Time};
+use moclib::qty::{MocQty, Hpx, Time, Frequency};
 
 #[derive(StructOpt, Debug)]
 pub enum Constants {
   #[structopt(name = "space")]
-  /// Provides iformation on HEALPix (used in Space MOCs)
+  /// Provides information on HEALPix (used in Space MOCs)
   Space,
   #[structopt(name = "time")]
-  /// Provides iformation on HEALPix (used in Space MOCs)
+  /// Provides information on time MOCs
   Time,
+  #[structopt(name = "freq")]
+  /// Provides information on frequency MOCs
+  Frequency,
 }
 
 impl Constants {
   pub fn exec(self) -> Result<(), Box<dyn Error>> {
     match self {
       Constants::Space => print_hpx_info(),
       Constants::Time => print_time_info(),
+      Constants::Frequency => print_freq_info(),
     }
     Ok(())
   }
@@ -71,3 +75,19 @@ fn print_time_info() {
     println!("   {:2} {:20}", depth, Time::<u64>::n_cells(depth as u8));
   }
 }
+
+fn print_freq_info() {
+  println!("Frequency MOCs");
+  println!();
+  println!("Index types:");
+  println!("- u16 (short), depth max = {}", Frequency::<u16>::MAX_DEPTH);
+  println!("- u32   (int), depth max = {}", Frequency::<u32>::MAX_DEPTH);
+  println!("- u64  (long), depth max = {}", Frequency::<u64>::MAX_DEPTH);
+  println!();
+  println!("Layers info:");
+  println!("{:>5} {:>20}",  "depth", "ncells");
+  for depth in 0..=Frequency::<u64>::MAX_DEPTH {
+    println!("   {:2} {:20}", depth, Frequency::<u64>::n_cells(depth as u8));
+  }
+}
+

crates/cli/src/convert.rs

@@ -7,7 +7,7 @@ use std::error::Error;
 
 use structopt::StructOpt;
 
-use moclib::qty::{Hpx, Time};
+use moclib::qty::{Frequency, Hpx, Time};
 use moclib::deser::json::{from_json_aladin, cellmoc2d_from_json_aladin};
 use moclib::deser::ascii::{from_ascii_ivoa, from_ascii_stream, moc2d_from_ascii_ivoa};
 use moclib::deser::fits::{
@@ -30,7 +30,8 @@ use super::input::{InputFormat, fmt_from_extension};
 pub enum MocType {
   SMOC,
   TMOC,
-  STMOC
+  STMOC,
+  FMOC,
 }
 impl FromStr for MocType {
   type Err = String;
@@ -40,7 +41,8 @@ impl FromStr for MocType {
       "moc" | "smoc" => Ok(MocType::SMOC),
       "tmoc" => Ok(MocType::TMOC),
       "stmoc" => Ok(MocType::STMOC),
-      _ => Err(format!("Unrecognized moc type. Actual: '{}'. Expected: 'moc (or smoc), 'tmoc' or 'stmoc'", s)),
+      "fmoc" => Ok(MocType::FMOC),
+      _ => Err(format!("Unrecognized moc type. Actual: '{}'. Expected: 'moc (or smoc), 'tmoc', 'fmoc' or 'stmoc'", s)),
     }
   }
 }
@@ -51,7 +53,7 @@ pub struct Convert {
   /// Path of the input MOC file (or stdin if equals "-")
   input: PathBuf,
   #[structopt(short = "t", long = "type")]
-  /// Input MOC type ('smoc', 'tmoc' or 'stmoc') required for 'ascii', 'json' ans 'stream' inputs; ignored for 'fits'
+  /// Input MOC type ('smoc', 'tmoc', 'fmoc' or 'stmoc') required for 'ascii', 'json' ans 'stream' inputs; ignored for 'fits'
   moc_type: Option<MocType>,
   #[structopt(short = "f", long = "format")]
   /// Format of the input MOC ('ascii', 'json', 'fits' or 'stream') [default: guess from the file extension]
@@ -124,6 +126,23 @@ pub fn exec<R: BufRead>(
       let cellrange_it = from_ascii_stream::<u64, Time::<u64>, _>(input)?;
       output.write_tmoc_possibly_auto_converting_from_u64(cellrange_it.ranges())
     },
+    // FMOC
+    (Some(MocType::FMOC), InputFormat::Ascii) => {
+      let mut input_str = String::new();
+      input.read_to_string(&mut input_str)?;
+      let cellcellranges = from_ascii_ivoa::<u64, Frequency::<u64>>(&input_str)?;
+      output.write_fmoc_possibly_auto_converting_from_u64(cellcellranges.into_cellcellrange_moc_iter().ranges())
+    },
+    (Some(MocType::FMOC), InputFormat::Json) => {
+      let mut input_str = String::new();
+      input.read_to_string(&mut input_str)?;
+      let cells = from_json_aladin::<u64, Frequency::<u64>>(&input_str)?;
+      output.write_fmoc_possibly_auto_converting_from_u64(cells.into_cell_moc_iter().ranges())
+    },
+    (Some(MocType::FMOC), InputFormat::Stream) => {
+      let cellrange_it = from_ascii_stream::<u64, Frequency::<u64>, _>(input)?;
+      output.write_fmoc_possibly_auto_converting_from_u64(cellrange_it.ranges())
+    },
     // ST-MOC
     (Some(MocType::STMOC), InputFormat::Ascii) => {
       let mut input_str = String::new();
@@ -140,7 +159,7 @@ pub fn exec<R: BufRead>(
     (Some(MocType::STMOC), InputFormat::Stream) => {
       Err(String::from("No stream format for ST-MOCs yet.").into())
     },
-    // FITS file (SMOC or TMOC or ST-MOC)
+    // FITS file (SMOC or TMOC or FMOC, or ST-MOC)
     (_, InputFormat::Fits) => {
       let fits_res = from_fits_ivoa(input)?;
       match fits_res {
@@ -161,6 +180,11 @@ pub fn exec<R: BufRead>(
                 STMocType::V2(moc) => output.write_stmoc(moc),
                 STMocType::PreV2(moc) => output.write_stmoc(moc),
               },
+            MocQtyType::Freq(moc) =>
+              match moc {
+                RMocType::Ranges(moc) => output.write_fmoc_possibly_converting_to_u64(moc),
+                RMocType::Cells(moc) => output.write_fmoc_possibly_converting_to_u64(moc.into_cell_moc_iter().ranges()),
+              },
           },
         MocIdxType::U32(moc) =>
           match moc {
@@ -179,6 +203,11 @@ pub fn exec<R: BufRead>(
                 STMocType::V2(moc) => output.write_stmoc(moc),
                 STMocType::PreV2(moc) => output.write_stmoc(moc),
               },
+            MocQtyType::Freq(moc) =>
+              match moc {
+                RMocType::Ranges(moc) => output.write_fmoc_possibly_converting_to_u64(moc),
+                RMocType::Cells(moc) => output.write_fmoc_possibly_converting_to_u64(moc.into_cell_moc_iter().ranges()),
+              },
           },
         MocIdxType::U64(moc) =>
           match moc {
@@ -197,6 +226,11 @@ pub fn exec<R: BufRead>(
                 STMocType::V2(moc) => output.write_stmoc(moc),
                 STMocType::PreV2(moc) => output.write_stmoc(moc),
               },
+            MocQtyType::Freq(moc) =>
+              match moc {
+                RMocType::Ranges(moc) => output.write_fmoc_possibly_converting_to_u64(moc),
+                RMocType::Cells(moc) => output.write_fmoc_possibly_converting_to_u64(moc.into_cell_moc_iter().ranges()),
+              },
           },
       }
     },

crates/cli/src/filter.rs

@@ -65,7 +65,7 @@ impl CsvArgs {
 pub enum Filter {
   /// Filter a file containing equatorial coordinates using a Space MOC
   Position(PositionFilter), // hpx sort (flag to filter on streaming mode?)
-  /// Filter a file containing a time using a Time MOC (NOT YET IMPLEMETNED)
+  /// Filter a file containing a time using a Time MOC (NOT YET IMPLEMENTED)
   Time(TimeFilter),
   // SpaceTime(SpaceTime) // TODO
 }

crates/cli/src/from.rs

@@ -12,7 +12,7 @@ use structopt::StructOpt;
 use healpix::nested::Layer;
 
 use moclib::{
-  qty::{MocQty, Hpx, Time},
+  qty::{MocQty, Hpx, Time, Frequency},
   elem::valuedcell::valued_cells_to_moc_with_opt,
   elemset::range::HpxRanges,
   moc::{
@@ -362,6 +362,31 @@ pub enum From {
     #[structopt(subcommand)]
     out: OutputFormat
   },
+  #[structopt(name = "freqval")]
+  /// Create a Frequency MOC from a list of frequency (in Hz, one per line).
+  FreqValue {
+    /// Depth of the created MOC, in `[0, 57]`.
+    depth: u8,
+    #[structopt(parse(from_os_str))]
+    /// The input file, use '-' for stdin
+    input: PathBuf,
+    #[structopt(subcommand)]
+    out: OutputFormat
+  },
+  #[structopt(name = "freqrange")]
+  /// Create a Time MOC from a list of frequency range (in Hz, one range per line, lower bound first, then upper bound).
+  FreqRange {
+    /// Depth of the created MOC, in `[0, 61]`.
+    depth: u8,
+    #[structopt(parse(from_os_str))]
+    /// The input file, use '-' for stdin
+    input: PathBuf,
+    #[structopt(short = "s", long = "separator", default_value = " ")]
+    /// Separator between time lower and upper bounds (default = ' ')
+    separator: String,
+    #[structopt(subcommand)]
+    out: OutputFormat
+  },
 }
 
 impl From {
@@ -952,7 +977,83 @@ impl From {
           RangeMOC2::from_ranges_and_fixed_depth_cells(tdepth, sdepth, reader.lines().filter_map(line2trpos), None)
         };
         out.write_stmoc(moc2.into_range_moc2_iter())
-      }
+      },
+      From::FreqValue {
+        depth,
+        input,
+        out
+      } => {
+        let line2freq = move |line: std::io::Result<String>| {
+          match line.map(|s| s.parse::<f64>()) {
+            Ok(Ok(f)) => Some(f),
+            Ok(Err(e)) => {
+              eprintln!("Error reading or parsing line: {:?}", e);
+              None
+            }
+            Err(e) => {
+              eprintln!("Error reading or parsing line: {:?}", e);
+              None
+            }
+          }
+        };
+        if input == PathBuf::from(r"-") {
+          let stdin = std::io::stdin();
+          out.write_fmoc_possibly_auto_converting_from_u64(
+            RangeMOC::<u64, Frequency::<u64>>::from_freq_in_hz(
+              depth, stdin.lock().lines().filter_map(line2freq), None
+            ).into_range_moc_iter()
+          )
+        } else {
+          let f = File::open(input)?;
+          let reader = BufReader::new(f);
+          out.write_fmoc_possibly_auto_converting_from_u64(
+            RangeMOC::<u64, Frequency::<u64>>::from_freq_in_hz(
+              depth, reader.lines().filter_map(line2freq), None
+            ).into_range_moc_iter()
+          )
+        }
+      },
+      From::FreqRange {
+        depth,
+        input,
+        separator,
+        out
+      } => {
+        fn line2tr(separator: &str, line: std::io::Result<String>) -> Result<Range<f64>, Box<dyn Error>> {
+          let line = line?;
+          let (fmin, fmax) = line.trim()
+            .split_once(&separator)
+            .ok_or_else(|| String::from("split on space failed."))?;
+          let fmin = fmin.parse::<f64>()?;
+          let fmax = fmax.parse::<f64>()?;
+          Ok(fmin..fmax)
+        }
+        let line2trange = move |line: std::io::Result<String>| {
+          match line2tr(&separator, line) {
+            Ok(trange) => Some(trange),
+            Err(e) => {
+              eprintln!("Error reading or parsing line: {:?}", e);
+              None
+            }
+          }
+        };
+        if input == PathBuf::from(r"-") {
+          let stdin = std::io::stdin();
+          out.write_fmoc_possibly_auto_converting_from_u64(
+            RangeMOC::<u64, Frequency::<u64>>::from_freq_ranges_in_hz(
+              depth, stdin.lock().lines().filter_map(line2trange), None
+            ).into_range_moc_iter()
+          )
+        } else {
+          let f = File::open(input)?;
+          let reader = BufReader::new(f);
+          out.write_fmoc_possibly_auto_converting_from_u64(
+            RangeMOC::<u64, Frequency::<u64>>::from_freq_ranges_in_hz(
+              depth, reader.lines().filter_map(line2trange), None
+            ).into_range_moc_iter()
+          )
+        }
+      },
       // ST-MOC from t-moc + s-moc (we can then create a complex ST-MOC by union of elementary ST-MOCs)
       // - e.g. multiple observation of the same area of the sky
       // - XMM ST-MOC (from list of observations)?