Support Tuple indices in Stan CSV Header Reader

src/stan/io/json/json_data_handler.hpp

@@ -16,7 +16,7 @@
 #include <utility>
 #include <vector>
 #include <boost/algorithm/string.hpp>
-#include <boost/regex.hpp>
+#include <regex>
 
 namespace stan {
 
@@ -172,8 +172,8 @@ class json_data_handler : public stan::json::json_handler {
    *  and contain only letters, numbers, or an underscore.
    */
   bool valid_varname(const std::string& name) {
-    static const boost::regex re("[a-zA-Z][a-zA-Z0-9_]*");
-    return regex_match(name, re);
+    static const std::regex re("[a-zA-Z][a-zA-Z0-9_]*");
+    return std::regex_match(name, re);
   }
 
   bool is_array_tuples(const std::vector<std::string>& keys) {

src/stan/io/stan_csv_reader.hpp

@@ -13,6 +13,22 @@
 namespace stan {
 namespace io {
 
+inline void prettify_stan_csv_name(std::string& variable) {
+  if (variable.find_first_of(":.") != std::string::npos) {
+    std::vector<std::string> parts;
+    boost::split(parts, variable, boost::is_any_of(":"));
+    for (auto& part : parts) {
+      int pos = part.find('.');
+      if (pos > 0) {
+        part[pos] = '[';
+        std::replace(part.begin(), part.end(), '.', ',');
+        part += "]";
+      }
+    }
+    variable = boost::algorithm::join(parts, ".");
+  }
+}
+
 // FIXME: should consolidate with the options from
 // the command line in stan::lang
 struct stan_csv_metadata {
@@ -192,11 +208,8 @@ class stan_csv_reader {
       std::getline(ss, token, ',');
       boost::trim(token);
 
-      int pos = token.find('.');
-      if (pos > 0 && prettify_name) {
-        token.replace(pos, 1, "[");
-        std::replace(token.begin(), token.end(), '.', ',');
-        token += "]";
+      if (prettify_name) {
+        prettify_stan_csv_name(token);
       }
       header[idx++] = token;
     }

src/stan/model/model_base.hpp

@@ -80,6 +80,11 @@ class model_base : public prob_grad {
                                bool include_tparams = true,
                                bool include_gqs = true) const = 0;
   /**
+   * WARNING: This function bakes in the assumption that the
+   * parameter values are rectangular. This is already not true
+   * for Tuple types, and will be fundamentally broken for ragged
+   * arrays or sparse matrices.
+   *
    * Set the dimensionalities of constrained parameters, transformed
    * parameters, and generated quantities.  The input sequence is
    * cleared and resized.  The dimensions of each parameter

src/stan/services/sample/standalone_gqs.hpp

@@ -16,34 +16,6 @@
 namespace stan {
 namespace services {
 
-/**
- * Find the names, dimensions of the model parameters.
- * Assembles vectors of name, dimensions for the variables
- * declared in the parameters block.
- *
- * @tparam Model type of model
- * @param[in] model model to query
- * @param[in, out] param_names sequence of parameter names
- * @param[in, out] param_dimss sequence of variable dimensionalities
- */
-template <class Model>
-void get_model_parameters(const Model &model,
-                          std::vector<std::string> &param_names,
-                          std::vector<std::vector<size_t>> &param_dimss) {
-  std::vector<std::string> all_param_names;
-  model.get_param_names(all_param_names, false, false);
-  std::vector<std::vector<size_t>> dimss;
-  model.get_dims(dimss, false, false);
-  // remove zero-size
-  for (size_t i = 0; i < all_param_names.size(); i++) {
-    auto &v = dimss[i];
-    if (std::find(v.begin(), v.end(), 0) == v.end()) {
-      param_names.emplace_back(all_param_names[i]);
-      param_dimss.emplace_back(dimss[i]);
-    }
-  }
-}
-
 /**
  * Given a set of draws from a fitted model, generate corresponding
  * quantities of interest which are written to callback writer.
@@ -112,6 +84,42 @@ int standalone_generate(const Model &model, const Eigen::MatrixXd &draws,
   return error_codes::OK;
 }
 
+/**
+ * DEPRECATED: This function assumes dimensions are rectangular,
+ * a restriction which the Stan language may soon relax.
+ *
+ * Find the names, dimensions of the model parameters.
+ * Assembles vectors of name, dimensions for the variables
+ * declared in the parameters block.
+ *
+ * @tparam Model type of model
+ * @param[in] model model to query
+ * @param[in, out] param_names sequence of parameter names
+ * @param[in, out] param_dimss sequence of variable dimensionalities
+ */
+template <class Model>
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((deprecated))
+#elif defined(_MSC_VER)
+__declspec(deprecated)
+#endif
+void get_model_parameters(const Model &model,
+                          std::vector<std::string> &param_names,
+                          std::vector<std::vector<size_t>> &param_dimss) {
+  std::vector<std::string> all_param_names;
+  model.get_param_names(all_param_names, false, false);
+  std::vector<std::vector<size_t>> dimss;
+  model.get_dims(dimss, false, false);
+  // remove zero-size
+  for (size_t i = 0; i < all_param_names.size(); i++) {
+    auto &v = dimss[i];
+    if (std::find(v.begin(), v.end(), 0) == v.end()) {
+      param_names.emplace_back(all_param_names[i]);
+      param_dimss.emplace_back(dimss[i]);
+    }
+  }
+}
+
 }  // namespace services
 }  // namespace stan
 #endif

src/test/unit/io/stan_csv_reader_test.cpp

@@ -12,6 +12,7 @@ class StanIoStanCsvReader : public testing::Test {
     metadata3_stream.open("src/test/unit/io/test_csv_files/metadata3.csv");
     header1_stream.open("src/test/unit/io/test_csv_files/header1.csv");
     header2_stream.open("src/test/unit/io/test_csv_files/header2.csv");
+    header3_stream.open("src/test/unit/io/test_csv_files/header3.csv");
     adaptation1_stream.open("src/test/unit/io/test_csv_files/adaptation1.csv");
     samples1_stream.open("src/test/unit/io/test_csv_files/samples1.csv");
 
@@ -29,6 +30,7 @@ class StanIoStanCsvReader : public testing::Test {
     metadata3_stream.close();
     header1_stream.close();
     header2_stream.close();
+    header3_stream.close();
     adaptation1_stream.close();
     samples1_stream.close();
 
@@ -43,6 +45,7 @@ class StanIoStanCsvReader : public testing::Test {
       samples1_stream;
   std::ifstream metadata3_stream, header2_stream;
   std::ifstream eight_schools_stream;
+  std::ifstream header3_stream;
 };
 
 TEST_F(StanIoStanCsvReader, read_metadata1) {
@@ -142,6 +145,40 @@ TEST_F(StanIoStanCsvReader, read_header2) {
   }
 }
 
+TEST_F(StanIoStanCsvReader, read_header_tuples) {
+  std::vector<std::string> header;
+  EXPECT_TRUE(
+      stan::io::stan_csv_reader::read_header(header3_stream, header, 0));
+
+  ASSERT_EQ(46, header.size());
+
+  // skip lp__ etc
+  int headeridx = 7;
+
+  for (int outer = 1; outer <= 3; ++outer) {
+    std::stringstream scalar;
+    scalar << "complicated[" << outer << "]"
+           << "." << 1;
+    EXPECT_EQ(scalar.str(), header[headeridx++]);
+
+    for (int inner = 1; inner <= 4; ++inner) {
+      std::stringstream inner_scalar;
+      inner_scalar << "complicated[" << outer << "]"
+                   << "." << 2 << "[" << inner << "]"
+                   << "." << 1;
+      EXPECT_EQ(inner_scalar.str(), header[headeridx++]);
+
+      for (int matrix_columns = 1; matrix_columns <= 2; ++matrix_columns) {
+        std::stringstream matrix;
+        matrix << "complicated[" << outer << "]"
+               << "." << 2 << "[" << inner << "]"
+               << "." << 2 << "[" << 1 << "," << matrix_columns << "]";
+        EXPECT_EQ(matrix.str(), header[headeridx++]);
+      }
+    }
+  }
+}
+
 TEST_F(StanIoStanCsvReader, read_adaptation1) {
   stan::io::stan_csv_adaptation adaptation;
   EXPECT_TRUE(stan::io::stan_csv_reader::read_adaptation(adaptation1_stream,

src/test/unit/io/test_csv_files/header3.csv

@@ -0,0 +1 @@
+lp__,accept_stat__,stepsize__,treedepth__,n_leapfrog__,divergent__,energy__,complicated.1:1,complicated.1:2.1:1,complicated.1:2.1:2.1.1,complicated.1:2.1:2.1.2,complicated.1:2.2:1,complicated.1:2.2:2.1.1,complicated.1:2.2:2.1.2,complicated.1:2.3:1,complicated.1:2.3:2.1.1,complicated.1:2.3:2.1.2,complicated.1:2.4:1,complicated.1:2.4:2.1.1,complicated.1:2.4:2.1.2,complicated.2:1,complicated.2:2.1:1,complicated.2:2.1:2.1.1,complicated.2:2.1:2.1.2,complicated.2:2.2:1,complicated.2:2.2:2.1.1,complicated.2:2.2:2.1.2,complicated.2:2.3:1,complicated.2:2.3:2.1.1,complicated.2:2.3:2.1.2,complicated.2:2.4:1,complicated.2:2.4:2.1.1,complicated.2:2.4:2.1.2,complicated.3:1,complicated.3:2.1:1,complicated.3:2.1:2.1.1,complicated.3:2.1:2.1.2,complicated.3:2.2:1,complicated.3:2.2:2.1.1,complicated.3:2.2:2.1.2,complicated.3:2.3:1,complicated.3:2.3:2.1.1,complicated.3:2.3:2.1.2,complicated.3:2.4:1,complicated.3:2.4:2.1.1,complicated.3:2.4:2.1.2

src/test/unit/services/sample/standalone_gqs_2390_test.cpp

@@ -41,19 +41,6 @@ TEST_F(ServicesStandaloneGQ4, genDraws_gq_test_vec_len_1) {
   csv_stream.close();
   EXPECT_EQ("b[1]", multidim_csv.header[7]);
 
-  // model bug_2390_model has 1 param, vector[1] b;
-  std::vector<std::string> param_names;
-  std::vector<std::vector<size_t>> param_dimss;
-  stan::services::get_model_parameters(*model, param_names, param_dimss);
-
-  EXPECT_EQ(param_names.size(), 1);
-  EXPECT_EQ(param_dimss.size(), 1);
-  int total = 1;
-  for (size_t i = 0; i < param_dimss[0].size(); ++i) {
-    total *= param_dimss[0][i];
-  }
-  EXPECT_EQ(total, 1);
-
   std::stringstream sample_ss;
   stan::callbacks::stream_writer sample_writer(sample_ss, "");
   int return_code = stan::services::standalone_generate(

src/test/unit/services/sample/standalone_gqs_bad_test.cpp

@@ -31,12 +31,6 @@ class ServicesStandaloneGQ2 : public testing::Test {
 };
 
 TEST_F(ServicesStandaloneGQ2, no_QoIs) {
-  std::vector<std::string> param_names;
-  std::vector<std::vector<size_t>> param_dimss;
-  stan::services::get_model_parameters(*model, param_names, param_dimss);
-  EXPECT_EQ(param_names.size(), 1);
-  EXPECT_EQ(param_dimss.size(), 1);
-
   const Eigen::MatrixXd draws(1, 1);
   std::stringstream sample_ss;
   stan::callbacks::stream_writer sample_writer(sample_ss, "");

src/test/unit/services/sample/standalone_gqs_multidim_test.cpp

@@ -48,19 +48,6 @@ TEST_F(ServicesStandaloneGQ2, genDraws_gq_test_multidim) {
   ASSERT_EQ(1000, multidim_csv.samples.rows());
   ASSERT_EQ(247, multidim_csv.samples.cols());
 
-  // model gq_test_multidim has 1 param, length 120
-  std::vector<std::string> param_names;
-  std::vector<std::vector<size_t>> param_dimss;
-  stan::services::get_model_parameters(*model, param_names, param_dimss);
-
-  EXPECT_EQ(param_names.size(), 1);
-  EXPECT_EQ(param_dimss.size(), 1);
-  int total = 1;
-  for (size_t i = 0; i < param_dimss[0].size(); ++i) {
-    total *= param_dimss[0][i];
-  }
-  EXPECT_EQ(total, 120);
-
   std::stringstream sample_ss;
   stan::callbacks::stream_writer sample_writer(sample_ss, "");
   int return_code = stan::services::standalone_generate(

src/test/unit/services/sample/standalone_gqs_test.cpp

@@ -47,14 +47,6 @@ TEST_F(ServicesStandaloneGQ, genDraws_bernoulli) {
   ASSERT_EQ(1000, bern_csv.samples.rows());
   ASSERT_EQ(19, bern_csv.samples.cols());
 
-  // model bernoulli.stan has 1 param
-  std::vector<std::string> param_names;
-  std::vector<std::vector<size_t>> param_dimss;
-  stan::services::get_model_parameters(*model, param_names, param_dimss);
-
-  EXPECT_EQ(param_names.size(), 1);
-  EXPECT_EQ(param_dimss.size(), 1);
-
   std::stringstream sample_ss;
   stan::callbacks::stream_writer sample_writer(sample_ss, "");
   int return_code = stan::services::standalone_generate(

src/test/unit/services/sample/standalone_gqs_zero_sized_test.cpp

@@ -30,20 +30,3 @@ class ServicesStandaloneGQ2 : public ::testing::Test {
   stan::callbacks::stream_logger logger;
   stan_model *model;
 };
-
-TEST_F(ServicesStandaloneGQ2, zero_sized_params_get_model_parameters) {
-  std::vector<std::string> param_names;
-  std::vector<std::vector<size_t>> param_dimss;
-  stan::services::get_model_parameters(*model, param_names, param_dimss);
-
-  EXPECT_EQ(param_names.size(), 4);
-  EXPECT_EQ(param_dimss.size(), 4);
-  EXPECT_EQ(param_dimss[0].size(), 0);
-  EXPECT_EQ(param_dimss[1].size(), 1);
-  EXPECT_EQ(param_dimss[1][0], 5);
-  EXPECT_EQ(param_dimss[2].size(), 1);
-  EXPECT_EQ(param_dimss[2][0], 6);
-  EXPECT_EQ(param_dimss[3].size(), 2);
-  EXPECT_EQ(param_dimss[3][0], 2);
-  EXPECT_EQ(param_dimss[3][1], 3);
-}