From b1701977191af648e12f7429387182f96cc9ceb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9=20Offringa?= <offringa@astron.nl>
Date: Thu, 27 Jan 2011 17:16:44 +0000
Subject: [PATCH] Bug 1491: added EoR ref file interface
---
.gitattributes | 5 +
CEP/DP3/AOFlagger/detectrfi.kdevelop | 2 +-
.../include/AOFlagger/CMakeLists.txt | 7 +
.../include/AOFlagger/ref/copyallscript.h | 94 +++++++++++++
.../AOFlagger/include/AOFlagger/ref/reffile.h | 92 +++++++++++++
.../include/AOFlagger/ref/reffileentry.h | 128 ++++++++++++++++++
.../include/AOFlagger/ref/reffileexception.h | 36 +++++
CEP/DP3/AOFlagger/src/CMakeLists.txt | 2 +
CEP/DP3/AOFlagger/src/aocopyallscript.cpp | 37 +++++
9 files changed, 402 insertions(+), 1 deletion(-)
create mode 100644 CEP/DP3/AOFlagger/include/AOFlagger/ref/copyallscript.h
create mode 100644 CEP/DP3/AOFlagger/include/AOFlagger/ref/reffile.h
create mode 100644 CEP/DP3/AOFlagger/include/AOFlagger/ref/reffileentry.h
create mode 100644 CEP/DP3/AOFlagger/include/AOFlagger/ref/reffileexception.h
create mode 100644 CEP/DP3/AOFlagger/src/aocopyallscript.cpp
diff --git a/.gitattributes b/.gitattributes
index 38b18605f50..b2f5a27619c 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -207,6 +207,10 @@ CEP/DP3/AOFlagger/include/AOFlagger/msio/timefrequencydata.h -text
CEP/DP3/AOFlagger/include/AOFlagger/msio/timefrequencyimager.h -text
CEP/DP3/AOFlagger/include/AOFlagger/msio/timefrequencymetadata.h -text
CEP/DP3/AOFlagger/include/AOFlagger/msio/types.h -text
+CEP/DP3/AOFlagger/include/AOFlagger/ref/copyallscript.h -text
+CEP/DP3/AOFlagger/include/AOFlagger/ref/reffile.h -text
+CEP/DP3/AOFlagger/include/AOFlagger/ref/reffileentry.h -text
+CEP/DP3/AOFlagger/include/AOFlagger/ref/reffileexception.h -text
CEP/DP3/AOFlagger/include/AOFlagger/rfi/antennaflagcountplot.h -text
CEP/DP3/AOFlagger/include/AOFlagger/rfi/eigenvalue.h -text
CEP/DP3/AOFlagger/include/AOFlagger/rfi/frequencyflagcountplot.h -text
@@ -327,6 +331,7 @@ CEP/DP3/AOFlagger/scripts/distributed/statistics.rfis -text
CEP/DP3/AOFlagger/scripts/package-without-lofarstman.sh -text
CEP/DP3/AOFlagger/scripts/package.sh -text
CEP/DP3/AOFlagger/src/CMakeLists.txt -text
+CEP/DP3/AOFlagger/src/aocopyallscript.cpp -text
CEP/DP3/AOFlagger/src/aofrequencyfilter.cpp -text
CEP/DP3/AOFlagger/src/cleaner.cpp -text
CEP/DP3/AOFlagger/src/colormapper.cpp -text
diff --git a/CEP/DP3/AOFlagger/detectrfi.kdevelop b/CEP/DP3/AOFlagger/detectrfi.kdevelop
index 768f36ed3d0..505bc9a525e 100644
--- a/CEP/DP3/AOFlagger/detectrfi.kdevelop
+++ b/CEP/DP3/AOFlagger/detectrfi.kdevelop
@@ -47,7 +47,7 @@
<default/>
</environments>
<prio>0</prio>
- <defaulttarget>aofrequencyfilter</defaulttarget>
+ <defaulttarget>aocopyallscript</defaulttarget>
<makeoptions>-s</makeoptions>
</make>
<blacklist/>
diff --git a/CEP/DP3/AOFlagger/include/AOFlagger/CMakeLists.txt b/CEP/DP3/AOFlagger/include/AOFlagger/CMakeLists.txt
index d458637c0c9..a7d98d844e8 100644
--- a/CEP/DP3/AOFlagger/include/AOFlagger/CMakeLists.txt
+++ b/CEP/DP3/AOFlagger/include/AOFlagger/CMakeLists.txt
@@ -102,6 +102,13 @@ install(FILES
msio/types.h
DESTINATION include/${PACKAGE_NAME}/msio)
+install(FILES
+ ref/copyallscript.h
+ ref/reffile.h
+ ref/reffileentry.h
+ ref/reffileexception.h
+ DESTINATION include/${PACKAGE_NAME}/ref)
+
install(FILES
rfi/antennaflagcountplot.h
rfi/frequencyflagcountplot.h
diff --git a/CEP/DP3/AOFlagger/include/AOFlagger/ref/copyallscript.h b/CEP/DP3/AOFlagger/include/AOFlagger/ref/copyallscript.h
new file mode 100644
index 00000000000..64361a1efb4
--- /dev/null
+++ b/CEP/DP3/AOFlagger/include/AOFlagger/ref/copyallscript.h
@@ -0,0 +1,94 @@
+/***************************************************************************
+ * Copyright (C) 2011 by A.R. Offringa *
+ * offringa@astro.rug.nl *
+ * *
+ * This program is free software; you can redistribute it and/or modify *
+ * it under the terms of the GNU General Public License as published by *
+ * the Free Software Foundation; either version 2 of the License, or *
+ * (at your option) any later version. *
+ * *
+ * This program is distributed in the hope that it will be useful, *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+ * GNU General Public License for more details. *
+ * *
+ * You should have received a copy of the GNU General Public License *
+ * along with this program; if not, write to the *
+ * Free Software Foundation, Inc., *
+ * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
+ ***************************************************************************/
+
+#ifndef AO_COPYALLSCRIPT_H
+#define AO_COPYALLSCRIPT_H
+
+#include <iostream>
+#include <map>
+#include <string>
+#include <vector>
+
+#include <AOFlagger/ref/reffile.h>
+
+namespace AOTools
+{
+ class CopyAllScript
+ {
+ public:
+ static void Make(std::ostream &stream, const std::string &refFilePath, const std::string &destination)
+ {
+ typedef std::vector<std::string> PathList;
+ typedef std::map<std::string, PathList> PathsPerNodeType;
+ typedef std::pair<std::string, PathList> NodeAndPaths;
+
+ RefFile file(refFilePath);
+
+ PathsPerNodeType pathsPerNode;
+
+ for(RefFile::const_iterator i = file.begin(); i != file.end() ; ++i)
+ {
+ if(pathsPerNode.count(i->Node()) != 0)
+ pathsPerNode.find(i->Node())->second.push_back(i->Path());
+ else
+ {
+ PathList newPathList;
+ newPathList.push_back(i->Path());
+ pathsPerNode.insert(NodeAndPaths(i->Node(), newPathList));
+ }
+ }
+
+ stream <<
+ "#! /bin/bash\n"
+ "# Created by aocopyallscript to move sets in \n# " << refFilePath << "\n"
+ "# to local path\n# " << destination << "\n"
+ "# Set contains " << file.Count() << " MS directories\n"
+ "# Number of nodes: " << pathsPerNode.size() << "\n";
+ for(PathsPerNodeType::const_iterator i=pathsPerNode.begin(); i!=pathsPerNode.end(); ++i)
+ {
+ const std::string node = i->first;
+ const PathList &paths = i->second;
+
+ stream
+ << "function copy_" << node << " {\n"
+ << " mkdir -p " << destination << "\n";
+ for(PathList::const_iterator p=paths.begin();p!=paths.end();++p)
+ {
+ const std::string &path = *p;
+ stream
+ << " echo " << path << " \\(" << node << "\\)\n"
+ << " ssh " << node << " -C \"cp -r " << path << " " << destination << "\"\n";
+ }
+ stream
+ << "}\n\n";
+ }
+ for(PathsPerNodeType::const_iterator i=pathsPerNode.begin(); i!=pathsPerNode.end(); ++i)
+ {
+ const std::string node = i->first;
+ stream << "copy_" << node << " &\n";
+ }
+ stream
+ << "wait\n"
+ << "echo All done.\n";
+ }
+ };
+}
+
+#endif // AO_COPYALLSCRIPT_H
diff --git a/CEP/DP3/AOFlagger/include/AOFlagger/ref/reffile.h b/CEP/DP3/AOFlagger/include/AOFlagger/ref/reffile.h
new file mode 100644
index 00000000000..075678d6942
--- /dev/null
+++ b/CEP/DP3/AOFlagger/include/AOFlagger/ref/reffile.h
@@ -0,0 +1,92 @@
+/***************************************************************************
+ * Copyright (C) 2011 by A.R. Offringa *
+ * offringa@astro.rug.nl *
+ * *
+ * This program is free software; you can redistribute it and/or modify *
+ * it under the terms of the GNU General Public License as published by *
+ * the Free Software Foundation; either version 2 of the License, or *
+ * (at your option) any later version. *
+ * *
+ * This program is distributed in the hope that it will be useful, *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+ * GNU General Public License for more details. *
+ * *
+ * You should have received a copy of the GNU General Public License *
+ * along with this program; if not, write to the *
+ * Free Software Foundation, Inc., *
+ * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
+ ***************************************************************************/
+
+#ifndef AO_REFFILE_H
+#define AO_REFFILE_H
+
+#include <fstream>
+#include <string>
+
+#include <AOFlagger/ref/reffileentry.h>
+#include <AOFlagger/ref/reffileexception.h>
+
+namespace AOTools
+{
+ class RefFile
+ {
+ public:
+ typedef std::vector<RefFileEntry>::const_iterator const_iterator;
+
+ RefFile()
+ {
+ }
+
+ explicit RefFile(const std::string &refFilePath)
+ {
+ Read(refFilePath);
+ }
+
+ void Read(const std::string &refFilePath)
+ {
+ _entries.clear();
+ _refFilePath = refFilePath;
+ std::ifstream file(_refFilePath.c_str());
+ RefFileEntry entry;
+ while(entry.read(file))
+ {
+ _entries.push_back(entry);
+ }
+ }
+
+ size_t Count() const
+ {
+ return _entries.size();
+ }
+
+ const RefFileEntry &operator[](const size_t index) const
+ {
+ return _entries[index];
+ }
+
+ const_iterator begin() const
+ {
+ return _entries.begin();
+ }
+
+ const_iterator end() const
+ {
+ return _entries.end();
+ }
+
+ private:
+ RefFile(const RefFile &) // don't allow copy
+ {
+ }
+
+ void operator=(const RefFile &) // don't allow assignment
+ {
+ }
+
+ std::vector<RefFileEntry> _entries;
+ std::string _refFilePath;
+ };
+}
+
+#endif //AO_REFFILE_H
diff --git a/CEP/DP3/AOFlagger/include/AOFlagger/ref/reffileentry.h b/CEP/DP3/AOFlagger/include/AOFlagger/ref/reffileentry.h
new file mode 100644
index 00000000000..082b42b3497
--- /dev/null
+++ b/CEP/DP3/AOFlagger/include/AOFlagger/ref/reffileentry.h
@@ -0,0 +1,128 @@
+/***************************************************************************
+ * Copyright (C) 2011 by A.R. Offringa *
+ * offringa@astro.rug.nl *
+ * *
+ * This program is free software; you can redistribute it and/or modify *
+ * it under the terms of the GNU General Public License as published by *
+ * the Free Software Foundation; either version 2 of the License, or *
+ * (at your option) any later version. *
+ * *
+ * This program is distributed in the hope that it will be useful, *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+ * GNU General Public License for more details. *
+ * *
+ * You should have received a copy of the GNU General Public License *
+ * along with this program; if not, write to the *
+ * Free Software Foundation, Inc., *
+ * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
+ ***************************************************************************/
+
+#ifndef AO_REFFILEENTRY_H
+#define AO_REFFILEENTRY_H
+
+#include <string>
+#include <sstream>
+
+#include <AOFlagger/ref/reffileexception.h>
+
+namespace AOTools
+{
+ class RefFileEntry
+ {
+ public:
+ friend class RefFile;
+
+ RefFileEntry() : _size(0)
+ {
+ }
+
+ RefFileEntry(const RefFileEntry &source) :
+ _path(source._path),
+ _frequency(source._frequency),
+ _size(source._size),
+ _node(source._node)
+ {
+ }
+
+ void operator=(const RefFileEntry &source)
+ {
+ _path = source._path;
+ _frequency = source._frequency;
+ _size = source._size;
+ _node = source._node;
+ }
+
+ const std::string &Path() const { return _path; }
+ const std::string &Frequency() const { return _frequency; }
+ unsigned Size() const { return _size; }
+ const std::string &Node() const { return _node; }
+
+ private:
+ std::string _path;
+ std::string _frequency;
+ unsigned _size;
+ std::string _node;
+
+ bool read(std::istream &stream)
+ {
+ std::string line;
+ do {
+ if(!stream.good()) return false;
+ std::getline(stream, line);
+ if(stream.fail()) return false;
+ if(stream.bad()) throw RefFileException("Error in IO");
+ } while(ignoreLine(line));
+
+ assignFromString(line);
+ return true;
+ }
+
+ void assignFromString(const std::string &line)
+ {
+ std::string::const_iterator i=line.begin();
+ if(!getNextToken(_path, i, line.end()))
+ throw RefFileException("Expecting a path");
+ if(!getNextToken(_frequency, i, line.end()))
+ throw RefFileException("Expecting frequency description");
+ std::string sizeString;
+ if(!getNextToken(sizeString, i, line.end()))
+ throw RefFileException("Expecting a size");
+ _size = atoi(sizeString.c_str());
+ if(!getNextToken(_node, i, line.end()))
+ throw RefFileException("Expecting a node");
+ }
+
+ static bool ignoreLine(const std::string &line)
+ {
+ for(std::string::const_iterator i=line.begin();i!=line.end();++i)
+ {
+ if(*i == '#') return true;
+ if(!ignorable(*i)) return false;
+ }
+ return true;
+ }
+
+ static bool ignorable(std::string::value_type ch)
+ {
+ return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r';
+ }
+
+ static bool getNextToken(std::string &dest, std::string::const_iterator &ptr, const std::string::const_iterator end)
+ {
+ std::ostringstream token;
+ while(ptr != end && ignorable(*ptr))
+ ++ptr;
+ if(ptr == end) return false;
+ while(ptr != end && !ignorable(*ptr))
+ {
+ token << *ptr;
+ ++ptr;
+ }
+ dest = token.str();
+ return dest.size() != 0;
+ }
+ };
+}
+
+#endif //AO_REFFILEENTRY_H
diff --git a/CEP/DP3/AOFlagger/include/AOFlagger/ref/reffileexception.h b/CEP/DP3/AOFlagger/include/AOFlagger/ref/reffileexception.h
new file mode 100644
index 00000000000..7076ea11adb
--- /dev/null
+++ b/CEP/DP3/AOFlagger/include/AOFlagger/ref/reffileexception.h
@@ -0,0 +1,36 @@
+/***************************************************************************
+ * Copyright (C) 2011 by A.R. Offringa *
+ * offringa@astro.rug.nl *
+ * *
+ * This program is free software; you can redistribute it and/or modify *
+ * it under the terms of the GNU General Public License as published by *
+ * the Free Software Foundation; either version 2 of the License, or *
+ * (at your option) any later version. *
+ * *
+ * This program is distributed in the hope that it will be useful, *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+ * GNU General Public License for more details. *
+ * *
+ * You should have received a copy of the GNU General Public License *
+ * along with this program; if not, write to the *
+ * Free Software Foundation, Inc., *
+ * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
+ ***************************************************************************/
+
+#ifndef AO_REFFILEEXCEPTION_H
+#define AO_REFFILEEXCEPTION_H
+
+#include <stdexcept>
+
+namespace AOTools
+{
+ class RefFileException : public std::runtime_error
+ {
+ public:
+ RefFileException() : std::runtime_error("Exception in reference file") { }
+ RefFileException(const std::string &message) : std::runtime_error(message) { }
+ };
+}
+
+#endif
diff --git a/CEP/DP3/AOFlagger/src/CMakeLists.txt b/CEP/DP3/AOFlagger/src/CMakeLists.txt
index 658eb373173..60ecf1d07b1 100644
--- a/CEP/DP3/AOFlagger/src/CMakeLists.txt
+++ b/CEP/DP3/AOFlagger/src/CMakeLists.txt
@@ -6,6 +6,8 @@ lofar_add_bin_program(rfihistory rfihistory.cpp)
lofar_add_bin_program(aofrequencyfilter aofrequencyfilter.cpp rfi/thresholdtools.cpp util/rng.cpp msio/image2d.cpp msio/fitsfile.cpp util/aologger.cpp)
+lofar_add_bin_program(aocopyallscript aocopyallscript.cpp)
+
set(GUI_PLOT_FILES
gui/plot/horizontalplotscale.cpp
gui/plot/plot2d.cpp
diff --git a/CEP/DP3/AOFlagger/src/aocopyallscript.cpp b/CEP/DP3/AOFlagger/src/aocopyallscript.cpp
new file mode 100644
index 00000000000..216d84ec879
--- /dev/null
+++ b/CEP/DP3/AOFlagger/src/aocopyallscript.cpp
@@ -0,0 +1,37 @@
+/***************************************************************************
+ * Copyright (C) 2011 by A.R. Offringa *
+ * offringa@astro.rug.nl *
+ * *
+ * This program is free software; you can redistribute it and/or modify *
+ * it under the terms of the GNU General Public License as published by *
+ * the Free Software Foundation; either version 2 of the License, or *
+ * (at your option) any later version. *
+ * *
+ * This program is distributed in the hope that it will be useful, *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+ * GNU General Public License for more details. *
+ * *
+ * You should have received a copy of the GNU General Public License *
+ * along with this program; if not, write to the *
+ * Free Software Foundation, Inc., *
+ * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
+ ***************************************************************************/
+
+#include <iostream>
+
+#include <AOFlagger/ref/copyallscript.h>
+
+using namespace std;
+
+int main(int argc, char *argv[])
+{
+ if(argc != 3)
+ {
+ cerr << "Syntax: " << argv[0] << " <reffile> <local-destination>\n";
+ return -1;
+ } else {
+ AOTools::CopyAllScript::Make(cout, argv[1], argv[2]);
+ return 0;
+ }
+}
--
GitLab