From d802d3a664509a7a66fe6ffec67b5ca336c2301d Mon Sep 17 00:00:00 2001 From: houstonjb Date: Tue, 4 Dec 2018 17:43:53 -0800 Subject: [PATCH 1/8] Update host.json --- FunctionAppCSVToJSON/host.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/FunctionAppCSVToJSON/host.json b/FunctionAppCSVToJSON/host.json index 1bb59ae..27084e4 100644 --- a/FunctionAppCSVToJSON/host.json +++ b/FunctionAppCSVToJSON/host.json @@ -1,4 +1,5 @@ { + "version": "2.0", "logger": { "categoryFilter": { "defaultLevel": "Information", @@ -9,4 +10,4 @@ } } } -} \ No newline at end of file +} From bb5ee87b409b558eea21b5697dca2278ac99861e Mon Sep 17 00:00:00 2001 From: Houston Brignano Date: Thu, 6 Dec 2018 16:19:04 -0800 Subject: [PATCH 2/8] Change it all to use csvHelper --- .gitignore | 5 +- FunctionAppCSVToJSON/CSVToJSON.cs | 101 ++++++------------ .../FunctionAppCSVToJSON.csproj | 7 +- 3 files changed, 42 insertions(+), 71 deletions(-) diff --git a/.gitignore b/.gitignore index 3c4efe2..3d15eb0 100644 --- a/.gitignore +++ b/.gitignore @@ -258,4 +258,7 @@ paket-files/ # Python Tools for Visual Studio (PTVS) __pycache__/ -*.pyc \ No newline at end of file +*.pyc + +#VSCODE stuff +.vscode \ No newline at end of file diff --git a/FunctionAppCSVToJSON/CSVToJSON.cs b/FunctionAppCSVToJSON/CSVToJSON.cs index 51bbadf..cb0773e 100644 --- a/FunctionAppCSVToJSON/CSVToJSON.cs +++ b/FunctionAppCSVToJSON/CSVToJSON.cs @@ -11,6 +11,8 @@ using System.Collections.Generic; using System.Linq; using System; +using CsvHelper; +using System.Text; namespace FunctionAppCSVToJSON { @@ -21,30 +23,13 @@ public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", { log.Info("C# HTTP trigger function CSVToJSON processed a request."); - char[] fieldSeperator = new char[] { ',' }; - - string fileName = req.Query["fileName"]; - string rowsToSkipStr = req.Query["rowsToSkip"]; + string fileName = req.Query["fileName"]; string errorMessage = ""; string requestBody = new StreamReader(req.Body).ReadToEnd(); dynamic data = JsonConvert.DeserializeObject(requestBody); - int rowsToSkip = 0; - long lineSkipCounter = 0; fileName = fileName ?? data?.fileName; - rowsToSkipStr = rowsToSkipStr ?? data?.rowsToSkip; - - if (rowsToSkipStr == null) - { - errorMessage = "Please pass a rowsToSkip on the query string or in the request body"; - log.Info("BadRequest: " + errorMessage); - return new BadRequestObjectResult(errorMessage); - } - else - { - Int32.TryParse(rowsToSkipStr, out rowsToSkip); - } if (fileName == null) { @@ -64,75 +49,57 @@ public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", log.Info("csv data is present."); - string[] csvLines = ToLines(csvData); + - log.Info(string.Format("There are {0} lines in the csv content {1}.", csvLines.Count(), fileName)); - - var headers = csvLines[0].Split(fieldSeperator).ToList(); + JsonResult resultSet = new JsonResult(fileName); + byte[] byteArray = Encoding.UTF8.GetBytes(csvData); + MemoryStream csvStream = new MemoryStream(byteArray); - foreach (var line in csvLines.Skip(rowsToSkip)) - { - //Check to see if a line is blank. - //This can happen on the last row if improperly terminated. - if (line != "" || line.Trim().Length > 0 ) - { - var lineObject = new JObject(); - var fields = line.Split(fieldSeperator); - - for (int x = 0; x < headers.Count; x++) - { - lineObject[headers[x]] = fields[x]; - } - - resultSet.Rows.Add(lineObject); - } - else - { - lineSkipCounter += 1; - } + var records = Convert(csvStream); + + JArray jsonarray = JArray.FromObject(records); + + foreach(var row in jsonarray.Children()){ + resultSet.Rows.Add(row); } - log.Info(string.Format("There were {0} lines skipped, not including the header row.", lineSkipCounter)); + + + log.Info(string.Format("There are {0} lines in the csv records content {1}.", records.Count(), fileName)); + + + return (ActionResult)new OkObjectResult(resultSet); } - private static string[] ToLines(string dataIn) - { - char[] EOLMarkerR = new char[] { '\r' }; - char[] EOLMarkerN = new char[] { '\n' }; - char[] EOLMarker = EOLMarkerR; - - //check to see if the file has both \n and \r for end of line markers. - //common for files comming from Unix\Linux systems. - if (dataIn.IndexOf('\n') > 0 && dataIn.IndexOf('\r') > 0) - { - //if we find both just remove one of them. - dataIn = dataIn.Replace("\n", ""); - } - //If the file only has \n then we will use that as the EOL marker to seperate the lines. - else if(dataIn.IndexOf('\n') > 0) - { - EOLMarker = EOLMarkerN; - } - - //How do we know the dynamic data will have Split capability? - return dataIn.Split(EOLMarker); + public static List Convert(Stream blob) + { + var sReader = new StreamReader(blob); + var csv = new CsvReader(sReader); + + csv.Read(); + csv.ReadHeader(); + + var csvRecords = csv.GetRecords().ToList(); + + return (csvRecords); } - } + public class JsonResult { public JsonResult(string fileName) { - Rows = new List(); + Rows = new JArray(); FileName = fileName; } public string FileName { get; set; } - public List Rows { get; set; } + public JArray Rows { get; set; } } } +} diff --git a/FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj b/FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj index f25dbef..b3486a7 100644 --- a/FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj +++ b/FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj @@ -1,10 +1,11 @@ - netstandard2.0 - v2 + netcoreapp2.1 + - + + From 4d94984ebe42b65258d615644e8aa0a29c7c1b13 Mon Sep 17 00:00:00 2001 From: Houston Brignano Date: Thu, 6 Dec 2018 16:25:21 -0800 Subject: [PATCH 3/8] remove rowsToSkip Requirement --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 20315c2..be0a77f 100644 --- a/README.md +++ b/README.md @@ -12,15 +12,12 @@ These instructions will get you a copy of the project up and running on your loc The JSON the function accepts consists of the following fields. -rowsToSkip - This indicates the number of rows to skip in the conversion process. - fileName - This is the source file where the data came from. This is passed in for downstream processes that may need to know which file was processed. csv - This is the raw data from the source file that you want to be converted to JSON. This content may contain \r\n or \n end of line markers. The function will detect them an process the csv data accordingly. ``` { - "rowsToSkip": 1, "fileName": "MyTestCSVFile.csv", "csv":"ID,Name,Score 1,Aaron,99 From da227d7509e2e110967f32ff068f359a85b84fa9 Mon Sep 17 00:00:00 2001 From: Houston Brignano Date: Fri, 7 Dec 2018 10:05:54 -0800 Subject: [PATCH 4/8] readme changes --- README.md | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index be0a77f..d171db6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Azure Function App [V2](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) that converts CSV data to JSON -I created this Azure Function as part of a larger solution to process incomming SFTP files. One possible soltion is to convert the CSV data to JSON and then perform Azure [SQL Bulk OPENROWSETS Bulk Inserts](https://blogs.msdn.microsoft.com/sqlserverstorageengine/2015/10/07/bulk-importing-json-files-into-sql-server/). +I Forked this from [aaronralls](https://github.com/aaronralls/FunctionAppCSVToJSON) originally and changed the convert function to leverage [csvhelper](https://joshclose.github.io/CsvHelper/). This Azure Function as part of a larger solution to process incomming SFTP files. ## Getting Started @@ -10,7 +10,7 @@ These instructions will get you a copy of the project up and running on your loc ### Sample Input -The JSON the function accepts consists of the following fields. +The JSON the function accepts consists of the following body fields. fileName - This is the source file where the data came from. This is passed in for downstream processes that may need to know which file was processed. @@ -60,10 +60,11 @@ rows - list of the CSV data in JSON format, with the field name from the header What things you need to install the software and how to install them ``` -Visual Studio 15.5.7 +Visual Studio Code Postman v6.0.7 +Azure Subscription +Azure funciton app service ``` -Download [Postman v6.0.7](https://www.getpostman.com/) ### Installing @@ -92,7 +93,7 @@ Explain how to run the automated tests for this system These variables used in the Postman tests. url - This is the URI of the Azure Function that you have published to or use for local testing. (ie: localhost:7071) -This variable is in each [test collection](https://github.com/aaronralls/FunctionAppCSVToJSON/Postman%20Tests). Be sure to update them both. +This variable is in each [test collection](https://github.com/houstonjb/FunctionAppCSVToJSON/Postman%20Tests). Be sure to update them both. ``` "variable": [ @@ -118,7 +119,7 @@ variable": [ ``` functions-key - This is the Function Key that you can use to limit access to your Azure functions. -This variable is only in the [Azure test collection](https://github.com/aaronralls/FunctionAppCSVToJSON/Postman%20Tests/FunctionAppCSVToJSON%20Azure.postman_collection.json). +This variable is only in the [Azure test collection](https://github.com/houstonjb/FunctionAppCSVToJSON/Postman%20Tests/FunctionAppCSVToJSON%20Azure.postman_collection.json). ``` "variable": [ @@ -135,17 +136,7 @@ This variable is only in the [Azure test collection](https://github.com/aaronral ### Break down into end to end tests -There are two [Postman collections](https://github.com/aaronralls/FunctionAppCSVToJSON/tree/master/Postman%20Tests) that cover the testing of the CSVToJSON function. One for local testing and the other for Azure testing. -#### Local Testing Collection - -FunctionAppCSVToJSON - -``` -POST to CSVToJSON with Windows text file contents - -Negative Test: POST to CSVToJSON with Windows text file contents, no filename -``` #### Azure Testing Collection @@ -161,25 +152,27 @@ CSVToJSON swagger ## Deployment -Add additional notes about how to deploy this on a live system +deploy to Azure function app using your favorite deployment methods. ## Built With - +.net core standard version 2.1 using Visual Studio Code and macOS. ## Contributing -Please read [CONTRIBUTING.md](https://gist.github.com/AaronRalls/b24679402957c63ec426) for details on our code of conduct, and the process for submitting pull requests to us. +send it. ## Versioning -We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [tags on this repository](https://github.com/AaronRalls/FunctionAppCSVToJSON/tags). +We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [releases on this repository](https://github.com/houstonjb/FunctionAppCSVToJSON/releases). ## Authors * **Aaron Ralls** - *Initial work* - [Aaron Ralls](https://github.com/AaronRalls) -See also the list of [contributors](https://github.com/AaronRalls/FunctionAppCSVToJSON/contributors) who participated in this project. +* **houstonjb** - *Modified* - [houstonjb](https://github.com/houstonjb) + +See also the list of [contributors](https://github.com/houstonjb/FunctionAppCSVToJSON/contributors) who participated in this project. ## License @@ -194,4 +187,3 @@ This project is licensed under the MIT License - see the [LICENSE.md](LICENSE) f ## Resources ## - [Azure Functions Documentation](https://docs.microsoft.com/en-us/azure/azure-functions/) -- Contact me on twitter [@cajunAA](https://www.twitter.com/cajunAA) From 475bf56fd6cb8963283d59ec8ccd619a1ec3b375 Mon Sep 17 00:00:00 2001 From: Houston Brignano Date: Fri, 7 Dec 2018 10:06:07 -0800 Subject: [PATCH 5/8] use ILogger instead of traceWriter --- FunctionAppCSVToJSON/CSVToJSON.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/FunctionAppCSVToJSON/CSVToJSON.cs b/FunctionAppCSVToJSON/CSVToJSON.cs index cb0773e..bcaadda 100644 --- a/FunctionAppCSVToJSON/CSVToJSON.cs +++ b/FunctionAppCSVToJSON/CSVToJSON.cs @@ -13,13 +13,14 @@ using System; using CsvHelper; using System.Text; +using Microsoft.Extensions.Logging; namespace FunctionAppCSVToJSON { public static class CSVToJSON { [FunctionName("CSVToJSON")] - public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]HttpRequest req, TraceWriter log) + public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]HttpRequest req, ILogger log) { log.Info("C# HTTP trigger function CSVToJSON processed a request."); From 26d34c482f68ac86ceb22748da102bbccb1cdcb7 Mon Sep 17 00:00:00 2001 From: Houston Brignano Date: Fri, 7 Dec 2018 10:06:49 -0800 Subject: [PATCH 6/8] logs --- FunctionAppCSVToJSON/CSVToJSON.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/FunctionAppCSVToJSON/CSVToJSON.cs b/FunctionAppCSVToJSON/CSVToJSON.cs index bcaadda..f502a6d 100644 --- a/FunctionAppCSVToJSON/CSVToJSON.cs +++ b/FunctionAppCSVToJSON/CSVToJSON.cs @@ -22,7 +22,7 @@ public static class CSVToJSON [FunctionName("CSVToJSON")] public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]HttpRequest req, ILogger log) { - log.Info("C# HTTP trigger function CSVToJSON processed a request."); + log.LogInformation("C# HTTP trigger function CSVToJSON processed a request."); string fileName = req.Query["fileName"]; string errorMessage = ""; @@ -35,7 +35,7 @@ public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", if (fileName == null) { errorMessage = "Please pass a fileName on the query string or in the request body"; - log.Info("BadRequest: " + errorMessage); + log.LogInformation("BadRequest: " + errorMessage); return new BadRequestObjectResult(errorMessage); } @@ -44,11 +44,11 @@ public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", if (csvData == null) { errorMessage = "Please pass the csv data in using the csv attribute in the request body"; - log.Info("BadRequest: " + errorMessage); + log.LogInformation("BadRequest: " + errorMessage); return new BadRequestObjectResult(errorMessage); } - log.Info("csv data is present."); + log.LogInformation("csv data is present."); @@ -69,7 +69,7 @@ public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", - log.Info(string.Format("There are {0} lines in the csv records content {1}.", records.Count(), fileName)); + log.LogInformation(string.Format("There are {0} lines in the csv records content {1}.", records.Count(), fileName)); From 6debfee522cf12fb2401276aca08608bce22e52c Mon Sep 17 00:00:00 2001 From: Houston Brignano Date: Fri, 7 Dec 2018 12:57:32 -0800 Subject: [PATCH 7/8] deal with bad data. --- FunctionAppCSVToJSON/CSVToJSON.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/FunctionAppCSVToJSON/CSVToJSON.cs b/FunctionAppCSVToJSON/CSVToJSON.cs index f502a6d..cbe7b99 100644 --- a/FunctionAppCSVToJSON/CSVToJSON.cs +++ b/FunctionAppCSVToJSON/CSVToJSON.cs @@ -59,6 +59,8 @@ public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", byte[] byteArray = Encoding.UTF8.GetBytes(csvData); MemoryStream csvStream = new MemoryStream(byteArray); + + log.LogInformation("translating CSV Data to a list"); var records = Convert(csvStream); JArray jsonarray = JArray.FromObject(records); @@ -76,12 +78,19 @@ public static IActionResult Run([HttpTrigger(AuthorizationLevel.Function, "get", return (ActionResult)new OkObjectResult(resultSet); } - public static List Convert(Stream blob) { + var sReader = new StreamReader(blob); var csv = new CsvReader(sReader); + //log bad data + csv.Configuration.BadDataFound = context => + { + + }; + + csv.Read(); csv.ReadHeader(); @@ -103,4 +112,6 @@ public JsonResult(string fileName) public JArray Rows { get; set; } } } + + } From 75cb892e997911d0a8383fda4083f2d57ada3e1d Mon Sep 17 00:00:00 2001 From: houstonjb Date: Thu, 13 Jun 2019 16:00:42 -0700 Subject: [PATCH 8/8] Update FunctionAppCSVToJSON.csproj --- FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj b/FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj index b3486a7..50336a1 100644 --- a/FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj +++ b/FunctionAppCSVToJSON/FunctionAppCSVToJSON.csproj @@ -1,11 +1,11 @@ - netcoreapp2.1 + netcoreapp2.2 - + @@ -16,4 +16,4 @@ Never - \ No newline at end of file +