CSVHelper 使用每个文件上的多个映射解析多个 CSV

Posted 2023-04-11

【中文标题】CSVHelper 使用每个文件上的多个映射解析多个 CSV

【英文标题】：CSVHelper Parsing Multiple CSVs using Multiple Mappings on each File

【发布时间】：2022-01-12 01:30:05

【问题描述】：

我正在尝试读取多个 CSV 文件，但在读取 CSV 文件之前，我不知道需要哪个 Classmap。我还需要使用这些映射来解释多个不同的标头名称。我的索引中目前有以下代码。有没有办法检查第一个字段“csv.GetField(0)”与每行的 3 个映射中的所有值，然后将该值添加到我的一个类列表中，然后添加到数据库中。我目前在我的 while 循环中收到一个错误，说列表中不存在三种记录类型之一，因为我认为它使用单个映射而不是多个映射会卡住

公共类 IndexModel : PageModel 私有只读 ClientContext _context; 公共 IndexModel(ClientContext 上下文) _context = 上下文；

    [BindProperty]
    public FileUpload fileUpload  get; set; 

    //create new lists for all class data types to add to database
    public List<Client> clients = new List<Client>();
    public List<Demographic> demographics = new List<Demographic>();
    public List<ReturnData> returnDatas = new List<ReturnData>();

    public void OnGet()
    
        ViewData["SuccessMessage"] = "Upload necessary files (MAX OF 4 FILES)";
    
    public ActionResult OnPostUpload(FileUpload fileUpload)
    
        //counter
        int i = 1;

        //type of Record
        RecordType type = RecordType.None;

        //for each file in the form files
        foreach (var file in fileUpload.FormFiles)
        
            //display filenames
            ViewData[i.ToString()] = file.FileName;
            i++;

            var config = new CsvConfiguration(CultureInfo.InvariantCulture)
            
                MissingFieldFound = null,
                HeaderValidated = null,
                IgnoreBlankLines = true,
                UseNewObjectForNullReferenceMembers = false
            ;

            //use streamReader and csvHelper to pull records from file
            using var stream = new MemoryStream();
            using var writer = new StreamWriter(stream);
            using var sreader = new StreamReader(file.OpenReadStream());
            using (var csv = new CsvReader(sreader, config))
            
                string[] headerRow = csv.HeaderRecord;

                //add all context mappings to interpret mutliple different header names
                csv.Context.RegisterClassMap<ClientMap>();
                csv.Context.RegisterClassMap<DemoMap>();
                csv.Context.RegisterClassMap<DataMap>();

                //grab records and add them to class lists
                while (csv.Read())
                
                    clients.Add(csv.GetRecord<Client>());
                    demographics.Add(csv.GetRecord<Demographic>());
                    returnDatas.Add(csv.GetRecord<ReturnData>());
                    

                
            
        

        foreach (var client in clients)
        
            _context.Client.Add(client);
            _context.SaveChanges();
        

        foreach (var demographic in demographics)
        
            _context.Demographic.Add(demographic);
            _context.SaveChanges();
        

        foreach (var returnData in returnDatas)
        
            _context.ReturnData.Add(returnData);
            _context.SaveChanges();
        

        //Save database changes
        _context.SaveChanges();

        //Process uploaded files
        ViewData["SuccessMessage"] = fileUpload.FormFiles.Count.ToString() + " file(s) uploaded!";
        var DropDownAndCheckBoxCount = i;
        return Page();
    
    public class FileUpload
    
        [Required]
        [Display(Name = "File")]
        public List<IFormFile> FormFiles  get; set;  // convert to list
        public string SuccessMessage  get; set; 
    



//Client data mapping
    public sealed class ClientMap : ClassMap<Client>

    public ClientMap()
    
        Map(m => m.ID).Name("ID", "Id");
        Map(m => m.FirstName).Name("FirstName", "First Name");
        Map(m => m.LastName).Name("LastName", "Last Name");
        Map(m => m.DoB).Name("DateOfBirth", "DoB", "Date of Birth");
        Map(m => m.Last4SS).Name("Last 4", "XXX-XX-1234", "Last Four", "Last 4 SS");
    


//enum to reference the different mappings for the csv
public enum RecordType

    None = 0,
    ClientType,
    DemographicType,
    ReturnDataType,
    TaxYearType


//Demographic data mapping
public sealed class DemoMap : ClassMap<Demographic>

    public DemoMap()
    
        Map(m => m.ID).Name("ID", "Id");
        Map(m => m.TaxYear).Name("Tax Year", "TaxYear");
        Map(m => m.Address).Name("Street Address", "Address");
        Map(m => m.Zip).Name("ZIP", "Zip", "zip", "Postal Code");
        Map(m => m.County).Name("County", "Location");
        Map(m => m.State).Name("State", "ST");
    


//ReturnData data mapping
public sealed class DataMap : ClassMap<ReturnData>

    public DataMap()
    
        Map(m => m.ID).Name("ID", "Id");
        Map(m => m.TaxYear).Name("TaxYear", "Tax Year");
        Map(m => m.FederalReturn).Name("FedReturn", "Federal");
        Map(m => m.TotalRefund).Name("Total Refund", "TotalRefund");
        Map(m => m.EITC).Name("EITC");
        Map(m => m.CTC).Name("CTC");
        Map(m => m.Dependents).Name("dependents");
        Map(m => m.SurveyScore).Name("Questions");
    

【参考方案1】：

如果您的 3 种不同的记录类型包含在每一行中，那么您正在执行的操作将起作用。您可以注册所有地图，并为该行中包含的每个记录数据获取一条记录。

void Main()

    var clients = new List<Client>();
    var demographics = new List<Demographic>();

    using (var reader = new StringReader("ClientId,First Name,DemoId,Tax Year\n1,John,4,2021"))
    using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture))
    
        csv.Context.RegisterClassMap<ClientMap>();
        csv.Context.RegisterClassMap<DemoMap>();

        while (csv.Read())
        
            clients.Add(csv.GetRecord<Client>());
            demographics.Add(csv.GetRecord<Demographic>());
        
    


public class Client

    public int ID  get; set; 
    public string FirstName  get; set; 


public class Demographic

    public int ID  get; set; 
    public int TaxYear  get; set; 


public sealed class ClientMap : ClassMap<Client>

    public ClientMap()
    
        Map(m => m.ID).Name("ClientId");
        Map(m => m.FirstName).Name("FirstName", "First Name");
    


public sealed class DemoMap : ClassMap<Demographic>

    public DemoMap()
    
        Map(m => m.ID).Name("DemoId");
        Map(m => m.TaxYear).Name("TaxYear", "Tax Year");
    

但是，听起来每个文件只包含一种类型的记录。因此，您需要一种方法来检测文件包含的记录类型并仅获取该记录类型。您可以使用某种标识符（例如记录类型的编号）开始每个文件，然后打开通过该标识符获取的记录类型。

void Main()

    var clients = new List<Client>();
    var demographics = new List<Demographic>();
    
    var files = new List<string>()
    
        "1\nId,First Name\n1,Jordan\n2,Jennifer",
        "2\nId,Tax Year\n1,2021\n2,2020"
    ;

    var config = new CsvConfiguration(CultureInfo.InvariantCulture)
    
        IgnoreBlankLines = true,
        UseNewObjectForNullReferenceMembers = false
    ;

    foreach (var file in files)
    
        using (var reader = new StringReader(file))
        using (var csv = new CsvReader(reader, config))
        
            csv.Context.RegisterClassMap<ClientMap>();
            csv.Context.RegisterClassMap<DemoMap>();

            csv.Read();

            switch (csv.GetField<RecordType>(0))
            
                case RecordType.ClientType:
                    clients.AddRange(csv.GetRecords<Client>().ToList());
                    break;
                case RecordType.DemographicType:
                    demographics.AddRange(csv.GetRecords<Demographic>().ToList());
                    break;
                default:
                    throw new Exception("Unable to determine record type");
            
        
    


public class Client

    public int ID  get; set; 
    public string FirstName  get; set; 


public class Demographic

    public int ID  get; set; 
    public int TaxYear  get; set; 


public sealed class ClientMap : ClassMap<Client>

    public ClientMap()
    
        Map(m => m.ID).Name("ID", "Id");
        Map(m => m.FirstName).Name("FirstName", "First Name");
    


public sealed class DemoMap : ClassMap<Demographic>

    public DemoMap()
    
        Map(m => m.ID).Name("ID", "Id");
        Map(m => m.TaxYear).Name("TaxYear", "Tax Year");
    


public enum RecordType

    None = 0,
    ClientType,
    DemographicType,
    ReturnDataType,
    TaxYearType

【讨论】：

我正在处理来自上传的 IFormFiles，我将如何解决这个问题而不是使用您在此处使用的 StringReader？

和以前一样。使用using var sreader = new StreamReader(file.OpenReadStream()); 而不是using (var reader = new StringReader(file))。我刚刚使用了StringReader，以便更轻松地创建一个示例，您可以快速测试而无需拥有文件。