release 0.6.0

- added csv2json argument to disable string trimming: starting and trailing whitespace characters will get removed from strings by default, ex. "foo, bar" will now yield "bar", not " bar", can be disabled with --no-trim - added support to cq for the same CSV related arguments as csv2json - added support for integer and floating point numbers when parsing CSV files - skip filenames passed to jq arguments, ex. --from-file
simonrupf · Jan 21, 2024 · 39d4968 · 39d4968
1 parent 0d8f4ab
commit 39d4968
Show file tree

Hide file tree

Showing 7 changed files with 102 additions and 14 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,15 @@
 Change Log of convert2json utilities
 ====================================
 
+Version 0.6.0 / 2023-01-21
+--------------------------
+- added csv2json argument to disable string trimming: starting and trailing
+  whitespace characters will get removed from strings by default, ex. "foo, bar"
+  will now yield "bar", not " bar", can be disabled with --no-trim
+- added support to cq for the same CSV related arguments as csv2json
+- added support for integer and floating point numbers when parsing CSV files
+- skip filenames passed to jq arguments, ex. --from-file
+
 Version 0.5.6 / 2023-01-13
 --------------------------
 - bump serde from 1.0.192 to 1.0.195

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -2,7 +2,7 @@
 name = "convert2json"
 description = "CLI utilities to convert CSV, TOML, XML & YAML into JSON and for use with jq."
 authors = ["Simon Rupf <simon@rupf.net>"]
-version = "0.5.6"
+version = "0.6.0"
 edition = "2021"
 license = "MIT"
 repository = "https://github.com/simonrupf/convert2json"

diff --git a/LICENSE.md b/LICENSE.md
@@ -788,7 +788,7 @@ SOFTWARE.
 
 #### Used by
 
-- [convert2json](https://github.com/simonrupf/convert2json) 0.5.6
+- [convert2json](https://github.com/simonrupf/convert2json) 0.6.0
 - [unsafe-libyaml](https://github.com/dtolnay/unsafe-libyaml) 0.2.10
 
 ```

diff --git a/README.md b/README.md
@@ -12,6 +12,22 @@ $ echo foo: bar | yaml2json
 {"foo":"bar"}
 $ tq -r .package.description Cargo.toml
 CLI utilities to convert CSV, TOML, XML & YAML into JSON on standard output or into jq.
+$ csv2json --help
+Usage: csv2json [-d <delimiter>] [-q <quote>] [-E <escape>] [--no-trim] [files...]
+
+Reads CSV from files or standard input and converts this to JSON, emitted on standard output. Any errors are reported to standard error and result in a non-zero exit code.
+
+Options:
+  -d, --delimiter   field delimiter to use when parsing CSV, defaults to: ,
+                    (comma)
+  -q, --quote       quote character to use when parsing CSV, defaults to: "
+                    (double quote)
+  -E, --escape      escape character to use when parsing CSV, to escape quote
+                    characters within a field. By default, quotes get escaped by
+                    doubling them.
+  --no-trim         do not trim headers & fields. By default, both get trimmed
+                    of starting or trailing whitespace characters.
+  --help            display usage information
 ```
 
 Overview
@@ -35,9 +51,6 @@ Alternatives:
 * [JavaScript 🌐](https://www.npmjs.com/search?q=yaml2json)
 * [Python 🐍](https://pypi.org/search/?q=yaml2json)
 
-To Do:
-- [ ] in jq arguments, ignore filenames if preceeded by certain flags (i.e. --from-file)
-
 Installation
 ------------
 Packages are provided (statically linked) for Debian & Ubuntu, as wells as RPM

diff --git a/src/csv.rs b/src/csv.rs
@@ -3,16 +3,12 @@ extern crate csv;
 
 use super::{exit, Error};
 use argh::FromArgs;
-use csv::ReaderBuilder;
-use serde::{Deserialize, Serialize};
+use csv::{ReaderBuilder, Trim};
 use std::collections::HashMap;
+use std::env::args;
 use std::io::Read;
 
-#[derive(Deserialize, Serialize)]
-pub struct CsvMap {
-    #[serde(flatten)]
-    values: HashMap<String, String>,
-}
+pub type CsvMap = HashMap<String, serde_json::Value>;
 
 #[derive(FromArgs)]
 /// Reads CSV from files or standard input and converts this to JSON, emitted on
@@ -31,6 +27,16 @@ struct CsvParameters {
     /// within a field. By default, quotes get escaped by doubling them.
     #[argh(option, short = 'E')]
     escape: Option<char>,
+
+    /// do not trim headers & fields. By default, both get trimmed of starting
+    /// or trailing whitespace characters.
+    #[argh(switch)]
+    no_trim: bool,
+
+    /// one or more CSV files to read
+    #[allow(dead_code)]
+    #[argh(positional)]
+    files: Vec<String>,
 }
 
 pub struct CsvReader {
@@ -52,6 +58,9 @@ impl CsvReader {
             // note that setting this to None would disable escape sequences entirely
             read.escape(Some(escape as u8)).double_quote(false);
         }
+        if !arguments.no_trim {
+            read.trim(Trim::All);
+        }
         Self { read }
     }
 
@@ -72,7 +81,35 @@ impl CsvReader {
 impl Default for CsvReader {
     fn default() -> Self {
         let mut read = ReaderBuilder::new();
+        let mut do_trim = true;
+        let mut read_var: i8 = -1;
+        let csv_args = ["-d", "-q", "-E"];
         read.flexible(true);
+        for arg in args().skip(1) {
+            if arg == "--no-trim" {
+                do_trim = false;
+            } else if read_var > -1 && read_var < 3 && arg.len() == 1 {
+                match read_var {
+                    0 => read.delimiter(arg.as_str().chars().next().unwrap() as u8),
+                    1 => read.quote(arg.as_str().chars().next().unwrap() as u8),
+                    2 => read
+                        .escape(Some(arg.as_str().chars().next().unwrap() as u8))
+                        .double_quote(false),
+                    _ => &mut read,
+                };
+                read_var = -1;
+            } else if csv_args.contains(&arg.as_str()) {
+                read_var = match csv_args.iter().position(|&flag| flag == arg) {
+                    Some(index) => index as i8,
+                    None => -1,
+                }
+            } else {
+                read_var = -1;
+            }
+        }
+        if do_trim {
+            read.trim(Trim::All);
+        }
         Self { read }
     }
 }
diff --git a/src/jq.rs b/src/jq.rs
@@ -62,14 +62,43 @@ pub fn parse_args() -> (Vec<String>, Vec<String>) {
     let mut arguments: Vec<String> = vec![];
     let mut files: Vec<String> = vec![];
     let mut args_done = false;
+    let mut skip_one = false;
+    let mut skip_file = false;
+    let mut skip_var = false;
+    let csv_args = ["-d", "-q", "-E"];
+    let file_args = [
+        "-f",
+        "--from-file",
+        "--run-tests",
+        "--slurpfile",
+        "--rawfile",
+    ];
+    let file_var_args = ["--slurpfile", "--rawfile"];
     for arg in args().skip(1) {
-        if args_done || Path::new(&arg).is_file() {
+        // ignore CSV arguments
+        if skip_one || arg == "--no-trim" {
+            skip_one = false;
+        } else if csv_args.contains(&arg.as_str()) {
+            skip_one = true;
+        } else if file_args.contains(&arg.as_str()) {
+            skip_file = true;
+            if file_var_args.contains(&arg.as_str()) {
+                skip_var = true;
+            }
+            arguments.push(arg);
+            continue;
+        } else if !skip_file && (args_done || Path::new(&arg).is_file()) {
             files.push(arg);
         } else if arg == "--" {
             args_done = true;
         } else {
             arguments.push(arg);
         }
+        if skip_var {
+            skip_var = false;
+        } else if skip_file {
+            skip_file = false;
+        }
     }
     (arguments, files)
 }