对象引用未设置为 Google Cloud Loadobject 中的对象实例

Posted 2023-03-24

【中文标题】对象引用未设置为 Google Cloud Loadobject 中的对象实例

【英文标题】：Object Reference Not Set to Instance of an Object in Google Cloud Loadobject

【发布时间】：2015-07-15 01:52:09

【问题描述】：

`loadconfig.SourceUris.Add(@"gs:\\planar-fulcrum-837\leadload-ip\01-         
 02-2013");`

设置为对象实例的空对象引用

【问题讨论】：

我不确定这里问的是什么......

我正在尝试将 .csv 文件从谷歌云加载到大查询表中。所以我使用 .net 云和大型查询库将云 .csv 文件加载到 bq。或者，如果有人可以建议如何将 MULTIPLE .CSV 文件从云插入到大查询...使用大查询 LOAD 命令行实用程序

为什么不首先问这个？另外，为什么是晦涩的标题？

【参考方案1】：

以下是将 CSV 文件从云存储加载到 Google Big Query 的工作示例。

更新“ServiceAccountEmail、KeyFileName、KeySecret、ProjectID、Dataset name等变量”。

将您的表架构添加到此变量中

TableSchema Schema = new TableSchema();

这里我使用的是单个文件加载，你可以在这个变量中添加 N 个 CSV 文件

System.Collections.Generic.IList<string> URIs = newSystem.Collections.Generic.List<string>();
URIs.Add(filePath);

使用下面的代码修改和使用它。祝你有美好的一天。 （我发现这个解决方案工作了 3 天以上）。

using Google.Apis.Auth.OAuth2;
using System.IO;
using System.Threading;
using Google.Apis.Bigquery.v2;
using Google.Apis.Bigquery.v2.Data;
using System.Data;
using Google.Apis.Services;
using System;
using System.Security.Cryptography.X509Certificates;

namespace GoogleBigQuery

    public class Class1
    
        private static void Main()
        
            try
            
                String serviceAccountEmail = "SERVICE ACCOUNT EMAIL";

                var certificate = new X509Certificate2(@"KEY FILE NAME & PATH", "KEY SECRET", X509KeyStorageFlags.Exportable);

                // SYNTAX: var certificate=new X509Certificate2(KEY FILE PATH+NAME (Here it resides in Bin\Debug folder so only name is enough), SECRET KEY, X509KeyStorageFlags.Exportable);

                ServiceAccountCredential credential = new ServiceAccountCredential(
                   new ServiceAccountCredential.Initializer(serviceAccountEmail)
                   
                       Scopes = new[]  BigqueryService.Scope.Bigquery, BigqueryService.Scope.BigqueryInsertdata, BigqueryService.Scope.CloudPlatform, BigqueryService.Scope.DevstorageFullControl 
                   .FromCertificate(certificate));

                //  Create and initialize the Bigquery service. Use the Project Name value
                //  from the New Project window for the ApplicationName variable.

                BigqueryService Service = new BigqueryService(new BaseClientService.Initializer()
                
                    HttpClientInitializer = credential,
                    ApplicationName = "APPLICATION NAME"
                );

                TableSchema Schema = new TableSchema();

                TableFieldSchema F1 = new TableFieldSchema();
                F1.Name = "COLUMN NAME";
                F1.Type = "STRING";
                F1.Mode = "REQUIRED";

                TableFieldSchema F2 = new TableFieldSchema();
                F1.Name = "COLUMN NAME";
                F1.Type = "INTEGER";
                F1.Mode = "NULLABLE";

                //Add N number of fields as per your needs

                System.Collections.Generic.IList<TableFieldSchema> FS = new System.Collections.Generic.List<TableFieldSchema>();
                FS.Add(F1);
                FS.Add(F2);

                Schema.Fields = FS;

                JobReference JR = JobUpload("PROJECT ID", "DATASET NAME", "TABLE NAME", @"gs://BUCKET NAME/FILENAME", Schema, "CREATE_IF_NEEDED", "WRITE_APPEND", '|', Service);

                //SYNTAX JobReference JR = JobUpload(PROJECT ID, DATASET NAME, TABLE NAME, FULL PATH OF CSV FILE,FILENAME IN CLOUD STORAGE, TABLE SCHEMA, CREATE DISPOSITION, DELIMITER, BIGQUERY SERVICE);

                while (true)
                
                    var PollJob = Service.Jobs.Get(JR.ProjectId, JR.JobId).Execute();

                    Console.WriteLine("Job status" + JR.JobId + ": " + PollJob.Status.State);
                    if (PollJob.Status.State.Equals("DONE"))
                    
                        Console.WriteLine("JOB Completed");
                        Console.ReadLine();
                        return;
                    
                
            
            catch (Exception e)
            
                Console.WriteLine("Error Occurred: " + e.Message);
            

            Console.ReadLine();
        

        public static JobReference JobUpload(string project, string dataset, string tableId, string filePath, TableSchema schema, string createDisposition, string writeDisposition, char delimiter, BigqueryService BigQueryService)
        

            TableReference DestTable = new TableReference();
            DestTable.ProjectId = project;
            DestTable.DatasetId = dataset;
            DestTable.TableId = tableId;

            Job Job = new Job();
            JobConfiguration Config = new JobConfiguration();
            JobConfigurationLoad ConfigLoad = new JobConfigurationLoad();

            ConfigLoad.Schema = schema;
            ConfigLoad.DestinationTable = DestTable;
            ConfigLoad.Encoding = "ISO-8859-1";
            ConfigLoad.CreateDisposition = createDisposition;
            ConfigLoad.WriteDisposition = writeDisposition;
            ConfigLoad.FieldDelimiter = delimiter.ToString();
            ConfigLoad.AllowJaggedRows = true;
            ConfigLoad.SourceFormat = "CSV";
            ConfigLoad.SkipLeadingRows = 1;
            ConfigLoad.MaxBadRecords = 100000;

            System.Collections.Generic.IList<string> URIs = new System.Collections.Generic.List<string>();
            URIs.Add(filePath);

            //You can add N number of CSV Files here

            ConfigLoad.SourceUris = URIs;
            Config.Load = ConfigLoad;
            Job.Configuration = Config;

            //set job reference (mainly job id)
            JobReference JobRef = new JobReference();
            Random r = new Random();
            var JobNo = r.Next();
            JobRef.JobId = "Job" + JobNo.ToString();
            JobRef.ProjectId = project;
            Job.JobReference = JobRef;

            JobsResource.InsertRequest InsertMediaUpload = new JobsResource.InsertRequest(BigQueryService, Job, Job.JobReference.ProjectId);
            var JobInfo = InsertMediaUpload.Execute();

            return JobRef;