diff --git a/CHANGELOG.md b/CHANGELOG.md index c82933c..ffa0cab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ Change Log of convert2json utilities ==================================== +Version 0.6.0 / 2023-01-21 +-------------------------- +- added csv2json argument to disable string trimming: starting and trailing + whitespace characters will get removed from strings by default, ex. "foo, bar" + will now yield "bar", not " bar", can be disabled with --no-trim +- added support to cq for the same CSV related arguments as csv2json +- added support for integer and floating point numbers when parsing CSV files +- skip filenames passed to jq arguments, ex. --from-file + Version 0.5.6 / 2023-01-13 -------------------------- - bump serde from 1.0.192 to 1.0.195 diff --git a/Cargo.lock b/Cargo.lock index 3efcf3d..11b8a29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,7 +35,7 @@ dependencies = [ [[package]] name = "convert2json" -version = "0.5.6" +version = "0.6.0" dependencies = [ "argh", "csv", diff --git a/Cargo.toml b/Cargo.toml index fccbe7d..af236c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "convert2json" description = "CLI utilities to convert CSV, TOML, XML & YAML into JSON and for use with jq." authors = ["Simon Rupf "] -version = "0.5.6" +version = "0.6.0" edition = "2021" license = "MIT" repository = "https://github.com/simonrupf/convert2json" diff --git a/LICENSE.md b/LICENSE.md index 87b61c4..58ba286 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -788,7 +788,7 @@ SOFTWARE. #### Used by -- [convert2json](https://github.com/simonrupf/convert2json) 0.5.6 +- [convert2json](https://github.com/simonrupf/convert2json) 0.6.0 - [unsafe-libyaml](https://github.com/dtolnay/unsafe-libyaml) 0.2.10 ``` diff --git a/README.md b/README.md index 0f53dc8..22f586b 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,22 @@ $ echo foo: bar | yaml2json {"foo":"bar"} $ tq -r .package.description Cargo.toml CLI utilities to convert CSV, TOML, XML & YAML into JSON on standard output or into jq. +$ csv2json --help +Usage: csv2json [-d ] [-q ] [-E ] [--no-trim] [files...] + +Reads CSV from files or standard input and converts this to JSON, emitted on standard output. Any errors are reported to standard error and result in a non-zero exit code. + +Options: + -d, --delimiter field delimiter to use when parsing CSV, defaults to: , + (comma) + -q, --quote quote character to use when parsing CSV, defaults to: " + (double quote) + -E, --escape escape character to use when parsing CSV, to escape quote + characters within a field. By default, quotes get escaped by + doubling them. + --no-trim do not trim headers & fields. By default, both get trimmed + of starting or trailing whitespace characters. + --help display usage information ``` Overview @@ -35,9 +51,6 @@ Alternatives: * [JavaScript 🌐](https://www.npmjs.com/search?q=yaml2json) * [Python 🐍](https://pypi.org/search/?q=yaml2json) -To Do: -- [ ] in jq arguments, ignore filenames if preceeded by certain flags (i.e. --from-file) - Installation ------------ Packages are provided (statically linked) for Debian & Ubuntu, as wells as RPM diff --git a/src/csv.rs b/src/csv.rs index 427f353..a7d1958 100644 --- a/src/csv.rs +++ b/src/csv.rs @@ -3,16 +3,12 @@ extern crate csv; use super::{exit, Error}; use argh::FromArgs; -use csv::ReaderBuilder; -use serde::{Deserialize, Serialize}; +use csv::{ReaderBuilder, Trim}; use std::collections::HashMap; +use std::env::args; use std::io::Read; -#[derive(Deserialize, Serialize)] -pub struct CsvMap { - #[serde(flatten)] - values: HashMap, -} +pub type CsvMap = HashMap; #[derive(FromArgs)] /// Reads CSV from files or standard input and converts this to JSON, emitted on @@ -31,6 +27,16 @@ struct CsvParameters { /// within a field. By default, quotes get escaped by doubling them. #[argh(option, short = 'E')] escape: Option, + + /// do not trim headers & fields. By default, both get trimmed of starting + /// or trailing whitespace characters. + #[argh(switch)] + no_trim: bool, + + /// one or more CSV files to read + #[allow(dead_code)] + #[argh(positional)] + files: Vec, } pub struct CsvReader { @@ -52,6 +58,9 @@ impl CsvReader { // note that setting this to None would disable escape sequences entirely read.escape(Some(escape as u8)).double_quote(false); } + if !arguments.no_trim { + read.trim(Trim::All); + } Self { read } } @@ -72,7 +81,35 @@ impl CsvReader { impl Default for CsvReader { fn default() -> Self { let mut read = ReaderBuilder::new(); + let mut do_trim = true; + let mut read_var: i8 = -1; + let csv_args = ["-d", "-q", "-E"]; read.flexible(true); + for arg in args().skip(1) { + if arg == "--no-trim" { + do_trim = false; + } else if read_var > -1 && read_var < 3 && arg.len() == 1 { + match read_var { + 0 => read.delimiter(arg.as_str().chars().next().unwrap() as u8), + 1 => read.quote(arg.as_str().chars().next().unwrap() as u8), + 2 => read + .escape(Some(arg.as_str().chars().next().unwrap() as u8)) + .double_quote(false), + _ => &mut read, + }; + read_var = -1; + } else if csv_args.contains(&arg.as_str()) { + read_var = match csv_args.iter().position(|&flag| flag == arg) { + Some(index) => index as i8, + None => -1, + } + } else { + read_var = -1; + } + } + if do_trim { + read.trim(Trim::All); + } Self { read } } } diff --git a/src/jq.rs b/src/jq.rs index 715b408..672e5e1 100644 --- a/src/jq.rs +++ b/src/jq.rs @@ -62,14 +62,43 @@ pub fn parse_args() -> (Vec, Vec) { let mut arguments: Vec = vec![]; let mut files: Vec = vec![]; let mut args_done = false; + let mut skip_one = false; + let mut skip_file = false; + let mut skip_var = false; + let csv_args = ["-d", "-q", "-E"]; + let file_args = [ + "-f", + "--from-file", + "--run-tests", + "--slurpfile", + "--rawfile", + ]; + let file_var_args = ["--slurpfile", "--rawfile"]; for arg in args().skip(1) { - if args_done || Path::new(&arg).is_file() { + // ignore CSV arguments + if skip_one || arg == "--no-trim" { + skip_one = false; + } else if csv_args.contains(&arg.as_str()) { + skip_one = true; + } else if file_args.contains(&arg.as_str()) { + skip_file = true; + if file_var_args.contains(&arg.as_str()) { + skip_var = true; + } + arguments.push(arg); + continue; + } else if !skip_file && (args_done || Path::new(&arg).is_file()) { files.push(arg); } else if arg == "--" { args_done = true; } else { arguments.push(arg); } + if skip_var { + skip_var = false; + } else if skip_file { + skip_file = false; + } } (arguments, files) }