Alexandria  2.16
Please provide a description of the project.
FileSystemProvider.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2012-2020 Euclid Science Ground Segment
3  *
4  * This library is free software; you can redistribute it and/or modify it under
5  * the terms of the GNU Lesser General Public License as published by the Free
6  * Software Foundation; either version 3.0 of the License, or (at your option)
7  * any later version.
8  *
9  * This library is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
11  * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
12  * details.
13  *
14  * You should have received a copy of the GNU Lesser General Public License
15  * along with this library; if not, write to the Free Software Foundation, Inc.,
16  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
26 #include <fstream>
27 #include <string>
28 #include <unordered_set>
29 #include <set>
30 #include <boost/filesystem.hpp>
31 #include <boost/algorithm/string.hpp>
33 #include "ElementsKernel/Logging.h"
35 #include "StringFunctions.h"
36 
37 namespace fs = boost::filesystem;
38 
39 namespace Euclid {
40 namespace XYDataset {
41 
42 static Elements::Logging logger = Elements::Logging::getLogger("FileSystemProvider");
43 
52 static std::vector<fs::path> getOrder(const fs::path& dir) {
53  std::vector<fs::path> result {};
54 
55  // First add the files in the order.txt
56  auto order_file = dir / "order.txt";
57  std::unordered_set<std::string> ordered_names {};
58  if (fs::exists(order_file)) {
59  std::ifstream in {order_file.c_str()};
60  while (in) {
61  std::string line;
62  getline(in, line);
63  size_t comment_pos = line.find('#');
64  if (comment_pos != std::string::npos) {
65  line = line.substr(0, comment_pos);
66  }
67  boost::trim(line);
68  if (!line.empty()) {
69  auto name = dir / line;
70  if (fs::exists(name)) {
71  result.emplace_back(name);
72  ordered_names.emplace(line);
73  } else {
74  logger.warn() << "Unknown name " << line << " in order.txt of " << dir << " directory";
75  }
76  }
77  }
78  }
79 
80  // Now we add any other files in the directory, which were not in the order.txt
81  // file. We use a set in order to avoid sorting problem between platforms.
82  std::set<fs::path> remaining_files {};
83  for (fs::directory_iterator iter {dir}; iter != fs::directory_iterator{}; ++ iter) {
84  if (ordered_names.count(iter->path().filename().string()) == 0) {
85  remaining_files.emplace(*iter);
86  }
87  }
88 
89  // Put the remaining files into the result vector
90  for (auto& file : remaining_files){
91  result.emplace_back(file);
92  }
93 
94  return result;
95 }
96 
97 
99  std::vector<fs::path> result {};
100  auto ordered_contents = getOrder(dir);
101  for (auto& name : ordered_contents) {
102  if (fs::is_directory(name)) {
103  auto sub_dir_contents = getRecursiveDirectoryContents(name);
104  result.insert(result.end(), sub_dir_contents.begin(), sub_dir_contents.end());
105  } else {
106  result.emplace_back(name);
107  }
108  }
109  return result;
110 }
111 
112 //-----------------------------------------------------------------------------
113 // Constructor
114 //-----------------------------------------------------------------------------
115 
117  : XYDatasetProvider(), m_root_path(root_path), m_parser(std::move(parser)) {
118 
119  std::vector<std::string> string_vector{};
120 
121  // Make sure the root path finishes with a "/" and only one
123 
124  // Convert path to boost filesytem object
125  fs::path fspath(m_root_path);
126  if (!fs::exists(fspath)) {
127  throw Elements::Exception() << "From FileSystemProvider: root path not found : "
128  << fspath;
129  }
130 
131  // Get all files below the root directory
132  if (fs::is_directory(fspath)) {
133  auto dir_contents = getRecursiveDirectoryContents(fspath);
134  for (auto& file : dir_contents) {
135  if (fs::is_regular_file(file) && m_parser->isDatasetFile(file.string()))
136  {
137  std::string dataset_name = m_parser->getName(file.string());
138  // Remove empty dataset name
139  if (dataset_name.empty()) {
140  continue;
141  }
142  // Remove the root part
143  std::string str = file.string();
144  str = str.substr(m_root_path.length(), str.length());
145  // Split by the character '/'
146  std::vector<std::string> groups {};
147  boost::split(groups, str, boost::is_any_of("/"));
148  // The last string is the file name, so we remove it
149  groups.pop_back();
150  QualifiedName qualified_name {groups, dataset_name};
151  // Fill up a map
152  auto ret = m_name_file_map.insert(make_pair(qualified_name, file.string()));
153  m_order_names.push_back(qualified_name);
154  // Check for unique record
155  if (!ret.second) {
156  throw Elements::Exception() << "Qualified name can not be inserted "
157  << "in the map. Qualify name : "
158  << qualified_name.qualifiedName()
159  << " Path :" << file.string();
160  }
161  }
162  }
163  }
164  else {
165  throw Elements::Exception() << " Root path : " << fspath.string() << " is not a directory!";
166  }
167 
168 }
169 
170 //-----------------------------------------------------------------------------
171 // listContents function
172 //-----------------------------------------------------------------------------
173 
175 
176  std::string my_group = group;
177  // Make sure the group finishes with a "/" and only one
178  while (!my_group.empty() && my_group.back() == '/') {
179  my_group.pop_back();
180  }
181  // Make sure the group do not start with a "/"
182  size_t pos = my_group.find_first_not_of("/");
183  if (!my_group.empty() && pos != 0) {
184  my_group = my_group.substr(pos);
185  }
186  if (!my_group.empty()) {
187  my_group.push_back('/');
188  }
189 
190  std::vector<QualifiedName> qualified_name_vector{};
191 
192  // Fill up vector with qualified name from the map
193  // Insert all qualified name where path contains the group name at the
194  // first position
195  for (auto qualified_name : m_order_names ) {
196  if (boost::starts_with(qualified_name.qualifiedName(), my_group)) {
197  qualified_name_vector.push_back(qualified_name);
198  }
199  } // Eof for
200 
201  return (qualified_name_vector);
202 }
203 
204 //-----------------------------------------------------------------------------
205 // getDataset function
206 //-----------------------------------------------------------------------------
207 
209 
210  auto it = m_name_file_map.find(qualified_name);
211  return (it != m_name_file_map.end()) ? m_parser->getDataset(it->second) : nullptr;
212 
213 }
214 
215 
217  auto it = m_name_file_map.find(qualified_name);
218  return (it != m_name_file_map.end()) ? m_parser->getParameter(it->second, key_word) : "";
219 }
220 
221 } /* namespace XYDataset */
222 } // end of namespace Euclid
T empty(T... args)
T find_first_not_of(T... args)
std::vector< QualifiedName > listContents(const std::string &group) override
List all files which belong to a group.
ELEMENTS_API auto split(Args &&... args) -> decltype(splitPath(std::forward< Args >(args)...))
STL namespace.
std::map< QualifiedName, std::string > m_name_file_map
std::string getParameter(const QualifiedName &qualified_name, const std::string &key_word) override
std::string checkEndSlashes(const std::string &input_str)
FileSystemProvider(const std::string &root_path, std::unique_ptr< FileParser > parser)
constructor The FileSystemProvider handles files in a directory tree.
static Elements::Logging logger
STL class.
T push_back(T... args)
void warn(const std::string &logMessage)
T pop_back(T... args)
std::unique_ptr< FileParser > m_parser
T find(T... args)
T length(T... args)
STL class.
STL class.
STL class.
std::vector< QualifiedName > m_order_names
T emplace(T... args)
T back(T... args)
T substr(T... args)
static std::vector< fs::path > getOrder(const fs::path &dir)
static std::vector< fs::path > getRecursiveDirectoryContents(const fs::path &dir)
This interface class provides the dataset following a qualified name object.
Represents a name qualified with a set of groups.
Definition: QualifiedName.h:66
static Logging getLogger(const std::string &name="")
STL class.
std::unique_ptr< XYDataset > getDataset(const QualifiedName &qualified_name) override
Get a dataset corresponding to an unique qualified name.