From 12f9ed10c291059ea1a8226370f7d21794381dd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Radek=20Pu=C5=A1?= <pusradek@fit.cvut.cz>
Date: Sun, 1 Dec 2019 22:01:54 +0100
Subject: [PATCH] fully implementedy csv import to DB (via web interface) + DI
---
.../ImportCSVLogic/IModelImporter.cs | 10 ++
.../ImportCSVLogic/ImportTester.cs | 91 +-----------------
.../ImportCSVLogic/ModelImporter.cs | 95 ++++++++++++++++++-
.../home/components/import/import.service.ts | 4 -
Core/Core/Controllers/ImportController.cs | 44 +++++++--
Core/Core/Startup.cs | 4 +
6 files changed, 144 insertions(+), 104 deletions(-)
create mode 100644 Core/Core/BusinessLogic/ImportCSVLogic/IModelImporter.cs
diff --git a/Core/Core/BusinessLogic/ImportCSVLogic/IModelImporter.cs b/Core/Core/BusinessLogic/ImportCSVLogic/IModelImporter.cs
new file mode 100644
index 0000000..4f19ce5
--- /dev/null
+++ b/Core/Core/BusinessLogic/ImportCSVLogic/IModelImporter.cs
@@ -0,0 +1,10 @@
+using System.Collections.Generic;
+
+namespace Core.BusinessLogic.ImportCSVLogic
+{
+ public interface IModelImporter
+ {
+ void Import(string filePath, string fileName, long userID);
+ void SaveRecords(long userID, string fileName, IEnumerable<DocumentLayout> records);
+ }
+}
\ No newline at end of file
diff --git a/Core/Core/BusinessLogic/ImportCSVLogic/ImportTester.cs b/Core/Core/BusinessLogic/ImportCSVLogic/ImportTester.cs
index 623fc36..fa5bd56 100644
--- a/Core/Core/BusinessLogic/ImportCSVLogic/ImportTester.cs
+++ b/Core/Core/BusinessLogic/ImportCSVLogic/ImportTester.cs
@@ -21,8 +21,6 @@ namespace Core.BusinessLogic.ImportCSVLogic
public void Test()
{
- IEnumerable<DocumentLayout> records;
-
//using (var reader = new StreamReader("C:\\Users\\Krypton0\\Desktop\\History_01_original.csv"))
//using (var reader = new StreamReader("C:\\Users\\Krypton0\\Desktop\\2178754183-0800_2019-01-01_2020-01-01.csv"))//csas
using (var reader = new StreamReader("C:\\Users\\Krypton0\\Desktop\\data-1565252156518.csv"))//csas
@@ -38,94 +36,9 @@ namespace Core.BusinessLogic.ImportCSVLogic
Console.WriteLine($"Field with names ['{string.Join("', '", headerNames)}'] at index '{index}' was not found.");
};
- records = csv.GetRecords<DocumentLayout>();
-
- SaveRecords(1,"testName" + DateTime.Now.ToString() + ".csv", records);
-
- string text = string.Empty;
- int i = 0;
-
- foreach (var doc in records)
- {
- text += doc.Dump(", ") + '\n';
- if (i++ > 10)
- break;
- }
-
- Console.WriteLine(text);
- }
- }
-
- private void SaveRecords(long userID, string fileName, IEnumerable<DocumentLayout> records)
- {
- User user = model.Users
- .Include(u => u.Files)
- .First(u => u.ID == userID);
-
- if (user == null)
- throw new ArgumentException($"UnknownUser with ID {userID}");
-
- if(user.Files.Any(f => f.Name == fileName))
- throw new ArgumentException($"Filename {fileName} already exists");
-
- Models.File file = new Models.File() { Name = fileName, Transactions = new List<Transaction>() };
-
- var transactionTypesList = model.TransactionTypes.ToList(); //DB performance
- var constantSymbolsList = model.ConstantSymbols.ToList();
- var accountList = model.Accounts.ToList();
-
- foreach(var record in records)
- {
- Transaction transaction = new Transaction(record);
-
- if (!string.IsNullOrEmpty(record.TransactionType))
- transaction.TransactionType = transactionTypesList.FirstOrDefault(t => record.TransactionType.StartsWith(t.Description));
-
- transaction.ConstantSymbol = ParseConstantSymbol(record.ConstantSymbol, constantSymbolsList);
-
- transaction.SenderAccount = ParseAccount(record.Account, accountList);
- if(transaction.SenderAccount == null)
- {
- Console.WriteLine($"Error parsing mandatory attribut SenderAccount: {record.Account}");
- continue;
- }
-
- transaction.RecipientAccount = ParseAccount(record.RecipientAccount, accountList);
-
-
- file.Transactions.Add(transaction);
+ ModelImporter importer = new ModelImporter(model);
+ importer.SaveRecords(1,"testName" + DateTime.Now.ToString() + ".csv", csv.GetRecords<DocumentLayout>());
}
- user.Files.Add(file);
- model.SaveChanges();
- }
-
- private Account ParseAccount(string accountToParse, List<Account> accountList)
- {
- Account parsedAccount = Account.CreateAccount(accountToParse);
- if (parsedAccount == null)
- return null;
-
- Account account = accountList.FirstOrDefault(a => a.Equals(parsedAccount));
- if (account != null)
- return account;
-
- model.Accounts.Add(parsedAccount);
- accountList.Add(parsedAccount);
- return parsedAccount;
- }
-
- private ConstantSymbol ParseConstantSymbol(string constantSymbolToParse, List<ConstantSymbol> constantSymbolsList)
- {
- if (!short.TryParse(constantSymbolToParse, out short parsedValue))
- return null;
-
- ConstantSymbol symbol = constantSymbolsList.FirstOrDefault(c => c.ProvidedID == parsedValue);
- if (symbol != null)
- return symbol;
-
- symbol = new ConstantSymbol(parsedValue);
- constantSymbolsList.Add(symbol);
- return symbol;
}
}
}
diff --git a/Core/Core/BusinessLogic/ImportCSVLogic/ModelImporter.cs b/Core/Core/BusinessLogic/ImportCSVLogic/ModelImporter.cs
index 62f81a0..eb73125 100644
--- a/Core/Core/BusinessLogic/ImportCSVLogic/ModelImporter.cs
+++ b/Core/Core/BusinessLogic/ImportCSVLogic/ModelImporter.cs
@@ -1,12 +1,16 @@
using Core.Data;
+using Core.Models;
+using CsvHelper;
+using Microsoft.EntityFrameworkCore;
using System;
using System.Collections.Generic;
+using System.IO;
using System.Linq;
using System.Threading.Tasks;
namespace Core.BusinessLogic.ImportCSVLogic
{
- public class ModelImporter
+ public class ModelImporter : IModelImporter
{
private readonly Model model;
@@ -15,9 +19,94 @@ namespace Core.BusinessLogic.ImportCSVLogic
this.model = model;
}
- public void Import()
+ public void Import(string filePath, string fileName, long userID)
{
- throw new NotImplementedException();
+ using (var reader = new StreamReader(filePath))
+ using (var csv = new CsvReader(reader))
+ {
+ csv.Configuration.BadDataFound = null;
+ csv.Configuration.HeaderValidated = null;
+ csv.Configuration.Delimiter = ",";
+ csv.Configuration.MissingFieldFound = (headerNames, index, context) =>
+ {
+ Console.WriteLine($"Field with names ['{string.Join("', '", headerNames)}'] at index '{index}' was not found.");
+ };
+
+ ModelImporter importer = new ModelImporter(model);
+ importer.SaveRecords(userID, fileName, csv.GetRecords<DocumentLayout>());
+ }
+ }
+
+ public void SaveRecords(long userID, string fileName, IEnumerable<DocumentLayout> records)
+ {
+ User user = model.Users
+ .Include(u => u.Files)
+ .First(u => u.ID == userID);
+
+ if (user == null)
+ throw new ArgumentException($"UnknownUser with ID {userID}");
+
+ if (user.Files.Any(f => f.Name == fileName))
+ throw new ArgumentException($"Filename {fileName} already exists");
+
+ Models.File file = new Models.File() { Name = fileName, Transactions = new List<Transaction>() };
+
+ var transactionTypesList = model.TransactionTypes.ToList(); //DB performance
+ var constantSymbolsList = model.ConstantSymbols.ToList();
+ var accountList = model.Accounts.ToList();
+
+ foreach (var record in records)
+ {
+ Transaction transaction = new Transaction(record);
+
+ if (!string.IsNullOrEmpty(record.TransactionType))
+ transaction.TransactionType = transactionTypesList.FirstOrDefault(t => record.TransactionType.StartsWith(t.Description));
+
+ transaction.ConstantSymbol = ParseConstantSymbol(record.ConstantSymbol, constantSymbolsList);
+
+ transaction.SenderAccount = ParseAccount(record.Account, accountList);
+ if (transaction.SenderAccount == null)
+ {
+ Console.WriteLine($"Error parsing mandatory attribut SenderAccount: {record.Account}");
+ continue;
+ }
+
+ transaction.RecipientAccount = ParseAccount(record.RecipientAccount, accountList);
+
+
+ file.Transactions.Add(transaction);
+ }
+ user.Files.Add(file);
+ model.SaveChanges();
+ }
+
+ private Account ParseAccount(string accountToParse, List<Account> accountList)
+ {
+ Account parsedAccount = Account.CreateAccount(accountToParse);
+ if (parsedAccount == null)
+ return null;
+
+ Account account = accountList.FirstOrDefault(a => a.Equals(parsedAccount));
+ if (account != null)
+ return account;
+
+ model.Accounts.Add(parsedAccount);
+ accountList.Add(parsedAccount);
+ return parsedAccount;
+ }
+
+ private ConstantSymbol ParseConstantSymbol(string constantSymbolToParse, List<ConstantSymbol> constantSymbolsList)
+ {
+ if (!short.TryParse(constantSymbolToParse, out short parsedValue))
+ return null;
+
+ ConstantSymbol symbol = constantSymbolsList.FirstOrDefault(c => c.ProvidedID == parsedValue);
+ if (symbol != null)
+ return symbol;
+
+ symbol = new ConstantSymbol(parsedValue);
+ constantSymbolsList.Add(symbol);
+ return symbol;
}
}
}
diff --git a/Core/Core/ClientApp/src/app/home/components/import/import.service.ts b/Core/Core/ClientApp/src/app/home/components/import/import.service.ts
index dbc8cdb..640d4dd 100644
--- a/Core/Core/ClientApp/src/app/home/components/import/import.service.ts
+++ b/Core/Core/ClientApp/src/app/home/components/import/import.service.ts
@@ -16,12 +16,8 @@ export class ImportService {
}
postFile(fileToUpload: File): Observable<Object> {
- //const endpoint = 'your-destination-url';
const formData: FormData = new FormData();
formData.append('fileKey', fileToUpload, fileToUpload.name);
return this.httpClient.post('api/Import', formData);
- //.post(endpoint, formData, { headers: yourHeadersConfig })
- //.map(() => { return true; })
- //.catch((e) => console.log(e));
}
}
diff --git a/Core/Core/Controllers/ImportController.cs b/Core/Core/Controllers/ImportController.cs
index 859ab03..fe34a09 100644
--- a/Core/Core/Controllers/ImportController.cs
+++ b/Core/Core/Controllers/ImportController.cs
@@ -2,7 +2,10 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
+using System.Security.Claims;
using System.Threading.Tasks;
+using Core.BusinessLogic.ImportCSVLogic;
+using Core.Data;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
@@ -14,6 +17,15 @@ namespace Core.Controllers
[Authorize(Policy = "RequireLoggedIn")]
public class ImportController : ControllerBase
{
+ private readonly Model Context;
+ private IModelImporter ModelImporter;
+
+ public ImportController(Model context, IModelImporter modelImporter)
+ {
+ Context = context;
+ ModelImporter = modelImporter;
+ }
+
// GET: api/Import/5
[HttpGet("{id}", Name = "GetImport")]
public string Get(int id)
@@ -23,33 +35,49 @@ namespace Core.Controllers
// POST: api/Import
[HttpPost]
+ [Authorize(Policy = "RequireLoggedIn")]
[RequestFormLimits(MultipartBodyLengthLimit = 268435456)]// Set the limit to 256 MB
- public async Task<IActionResult> Post([FromForm] List<IFormFile> files)
+ public async Task<IActionResult> Post()
{
+ if (Request == null || Request.Form == null || Request.Form.Files.Count < 1)
+ return BadRequest("No file present");
+
+ var files = Request.Form.Files;
+ Claim userJwtID = User.Claims.First(c => c.Type == "UserID");
+ long userID = long.Parse(userJwtID.Value);
long size = files.Sum(f => f.Length);
+ // Don't rely on or trust the FileName property without validation.
var filePaths = new List<string>();
- foreach (var formFile in files)
+ List<Task> asyncTasks = new List<Task>(files.Count);
+ foreach (IFormFile formFile in files)
{
if (formFile.Length <= 0)
continue;
// full path to file in temp location
- var filePath = Path.GetTempFileName();
+ string filePath = Path.GetTempFileName();
filePaths.Add(filePath);
- using (var stream = new FileStream(filePath, FileMode.Create))
- {
- await formFile.CopyToAsync(stream);
- }
+ asyncTasks.Add(SaveAndImport(formFile, filePath, userID));
}
// process uploaded files
- // Don't rely on or trust the FileName property without validation.
+ foreach (var task in asyncTasks)
+ await task;
return Ok(new { count = files.Count, size, filePaths });
}
+
+ private async Task SaveAndImport(IFormFile formFile, string filePath, long userID)
+ {
+ using (var stream = new FileStream(filePath, FileMode.Create))
+ {
+ await formFile.CopyToAsync(stream);
+ }
+ ModelImporter.Import(filePath, formFile.FileName, userID);
+ }
// DELETE: api/ApiWithActions/5
[HttpDelete("{id}")]
diff --git a/Core/Core/Startup.cs b/Core/Core/Startup.cs
index 088c819..6e739f4 100644
--- a/Core/Core/Startup.cs
+++ b/Core/Core/Startup.cs
@@ -1,3 +1,4 @@
+using Core.BusinessLogic.ImportCSVLogic;
using Core.Data;
using Core.Helpers;
using Microsoft.AspNetCore.Authentication.JwtBearer;
@@ -64,6 +65,9 @@ namespace Core
});
services.AddAuthorization( opt => opt.AddPolicy("RequireLoggedIn",policy => policy.RequireAuthenticatedUser()));
+
+ // DI
+ services.AddScoped<IModelImporter, ModelImporter>();
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
--
GitLab