具有大量数据的嵌套foreach会导致Outofmemory异常

Question

我正在处理机器返回的数据。该机器通常用于大型农场，它将在不同深度的每个点收集数据并使用不同的设备。

每个服务器场可能包含70-100个字段，每个字段包含大约1000万个数据

在这里，我需要处理每个数据并根据客户要求应用一些计算。

我需要首先遍历Farms然后在farm下面的字段，然后用于每个服务器场的设备然后根据深度记录每个设备。所以最后我希望我做了大约10亿次迭代。我的代码如下所示

    public async void MigrateData()
        {
            masterData = await CatalogService.ExportMasterData(AppDataModel.Catalog, this.UserId);

            foreach (var fieldItem in masterData.Fields)
            {
                var fieldReferenceId = fieldItem.ReferenceId;

                dynamic loggedData = AppDataModel.Documents.LoggedData.Where(data => data.FieldId == fieldReferenceId);
                dynamic fieldDbMappingData = fieldItem;

                foreach (var data in loggedData)
                {
                    yieldMaster.OperationalLogModalResponse = await YieldDataMigrationService.AddOperationalLogs("loggedDataDescription");
                    yieldMaster.OperationalLogDataModelResponse = await YieldDataMigrationService.AddOperationalLogData(fieldDbMappingData, yieldMaster.OperationalLogModalResponse);

                    FetchContentData(data);

                }

            }

        }
 private async void FetchContentData(LoggedData data)
        {

            foreach (var opdata in data.OperationData)
            {
               var filteredList = AggregateDataBasedOnFilter(opdata);

                int count = filteredList.Count;
                totalRecordCount += count;
                for (int i = 1; i <= count; i = i + 1000)
                {
                    var response = await YieldDataMigrationService.AddYiledData(filteredList.GetRange(i, i + 1000 >= count ? count - i : 1000));

                }
            }
            //Filter data based on timestamp values , get the first data in 5 seconds interval

            System.GC.Collect();
        } 

 private dynamic AggregateDataBasedOnFilter(OperationData opdata)
        {
            List<dynamic> listSpacialRecords = new List<dynamic>();
            IEnumerable<SpatialRecord> spacialRecords = opdata.GetSpatialRecords();
            spacialRecords = GetAggregateBasedOnTimeStamb(spacialRecords);
            Nullable<Guid> productid;
            for (int depth = 0; depth <= opdata.MaxDepth; depth++)
            {
                IEnumerable<DeviceElementUse> deviceElementUses = opdata.GetDeviceElementUses(depth);
                StevProduct productDbMappingData = masterData.Products.Where(product => product.ReferenceId == opdata.ProductId).FirstOrDefault();
                if (productDbMappingData == null)
                {
                    productid = null;
                }
                else
                {
                    productid = productDbMappingData.Id;
                }
                foreach (var deviceElement in deviceElementUses)
                {

                    List<dynamic> dvList = new List<dynamic>();
                    IEnumerable<WorkingData> workingData = deviceElement.GetWorkingDatas();

                    //foreach (var spacerecord in spacialRecords)
                    Parallel.ForEach(spacialRecords, (spacerecord) =>
                   {
                       List<MeterValue> dat = new List<MeterValue>();
                       var latitude = ((AgGateway.ADAPT.ApplicationDataModel.Shapes.Point)spacerecord.Geometry).Y;
                       var longitude = ((AgGateway.ADAPT.ApplicationDataModel.Shapes.Point)spacerecord.Geometry).X;
                       var timeStamp = spacerecord.Timestamp;
                       //Parallel.ForEach(workingData, (wdItem) =>
                       foreach (var wdItem in workingData)
                       {
                           RepresentationValue spaceMeteredValue = spacerecord.GetMeterValue(wdItem);
                           if (spaceMeteredValue != null && wdItem.Representation != null)
                           {
                               //row[wdItem.Representation.Code] = meteredValue.Value.Value;
                               var objMeterValue = new MeterValue();
                               objMeterValue.key = wdItem.Representation.Code;
                               objMeterValue.value = spaceMeteredValue.Designator != null ? Convert.ToString(spaceMeteredValue.Designator) : "";
                               dat.Add(objMeterValue);
                           }
                       }
                       var newSpacialvalue = new
                       {
                           operationLogDataId = yieldMaster.OperationalLogDataModelResponse.Id,
                           order = deviceElement.Order,
                           totalDistanceTravelled = deviceElement.TotalDistanceTravelled,
                           totalElapsedTime = deviceElement.TotalElapsedTime,
                           uploadedOn = DateTime.Now.ToUniversalTime(),
                           collectedOn = timeStamp.ToUniversalTime(),
                           cropId = "8296e610-c055-11e7-851e-ad7650a5f99c",
                           productId = productid,
                           latitude = latitude,
                           longitude = longitude,
                           deviceConfigurationId = deviceElement.DeviceConfigurationId,
                           operationDataId = deviceElement.OperationDataId,
                           spatialRecords = dat,
                           depth = depth,
                           timeStamp = timeStamp,
                           totaldata = totalRecordCount
                       };
                       lock (listSpacialRecords)
                       {
                           listSpacialRecords.Add(newSpacialvalue);
                       }

                   });


                }
            }

            listSpacialRecords = listSpacialRecords
                  .Skip(1)
                  .Aggregate(
                      listSpacialRecords.Take(1).ToList(),
                      (a, x) =>
                      {
                          if (x.timeStamp.Subtract(a.Last().timeStamp).TotalSeconds >= 10.0)
                          {
                              a.Add(x);
                          }
                          return a;
                      });
            GC.Collect();
            return listSpacialRecords;

        }

我的真实场景比这更复杂。它有很多foreach和计算。整个过程运行超过30分钟。但在我之间我的内存异常。不知道如何处理这么多庞大的数据。

任何人都有比嵌套的foreach更好的方法？或者任何避免内存不足的解决方案？

注意：我已将每个外观移动到单独的函数中，但仍然显示内存不足错误。我还有一个本地列表，处理由逻辑部分计算的数据。本地列表它不是全局对象