c#：实现动态编译，并实现动态MutilProcess功能（来自python mutilprocess的想法）

Posted 2020-09-29

　　由于之前一直遇到一些关于并行进行数据处理的时效果往往不好，不管是c#还是java程序都是一样，但是在Python中通过mutilprocess实现同样的功能时，却发现确实可以提高程序运行的性能，及服务器资源使用提高。python具体性能及mutilprocess用法，请参考：《Python：使用pymssql批量插入csv文件到数据库测试》

　　如有转载请标明原文地址：https://i.cnblogs.com/EditPosts.aspx?postid=7228337

　　很久之前就设想如何在c#中实现多进程的方案，如今终于设计出了C#中动态实现多进程的方案：

具体代码：

  1 using Microsoft.CSharp;
  2 using System;
  3 using System.CodeDom;
  4 using System.CodeDom.Compiler;
  5 using System.Collections.Generic;
  6 using System.Linq;
  7 using System.Reflection;
  8 using System.Text;
  9 
 10 namespace MutilProcessDemo
 11 {
 12     public class Program
 13     {
 14         static System.Timers.Timer timer = null;
 15 
 16         public static void Main(string[] args)
 17         {
 18             GenerateExe();
 19 
 20             timer = new System.Timers.Timer();
 21             timer.AutoReset = true;
 22             timer.Enabled = true;
 23             timer.Interval = 5000;
 24             timer.Elapsed += Timer_Elapsed;
 25 
 26 
 27             Console.WriteLine("Press Enter key to exit...");
 28             Console.ReadKey();
 29         }
 30 
 31         private static void DynamicCompiler()
 32         {
 33             // 1.CSharpCodePrivoder 
 34             CSharpCodeProvider objCSharpCodePrivoder = new CSharpCodeProvider();
 35 
 36             // 2.ICodeComplier 
 37             ICodeCompiler objICodeCompiler = objCSharpCodePrivoder.CreateCompiler();
 38 
 39             // 3.CompilerParameters 
 40             CompilerParameters objCompilerParameters = new CompilerParameters();
 41             objCompilerParameters.ReferencedAssemblies.Add("System.dll");
 42             objCompilerParameters.GenerateExecutable = false;
 43             objCompilerParameters.GenerateInMemory = true;
 44 
 45             // 4.CompilerResults 
 46             CompilerResults cr = objICodeCompiler.CompileAssemblyFromSource(objCompilerParameters, GenerateCode());
 47 
 48             if (cr.Errors.HasErrors)
 49             {
 50                 Console.WriteLine("编译错误：");
 51                 foreach (CompilerError err in cr.Errors)
 52                 {
 53                     Console.WriteLine(err.ErrorText);
 54                 }
 55             }
 56             else
 57             {
 58                 // 通过反射，调用HelloWorld的实例 
 59                 Assembly objAssembly = cr.CompiledAssembly;
 60                 object objHelloWorld = objAssembly.CreateInstance("DynamicCodeGenerate.HelloWorld");
 61                 MethodInfo objMI = objHelloWorld.GetType().GetMethod("OutPut");
 62 
 63                 Console.WriteLine(objMI.Invoke(objHelloWorld, null));
 64             }
 65 
 66             Console.ReadLine();
 67         }
 68 
 69         static string GenerateCode()
 70         {
 71             StringBuilder sb = new StringBuilder();
 72             sb.Append("using System;");
 73             sb.Append(Environment.NewLine);
 74             sb.Append("namespace DynamicCodeGenerate");
 75             sb.Append(Environment.NewLine);
 76             sb.Append("{");
 77             sb.Append(Environment.NewLine);
 78             sb.Append(" public class HelloWorld");
 79             sb.Append(Environment.NewLine);
 80             sb.Append(" {");
 81             sb.Append(Environment.NewLine);
 82             sb.Append(" public string OutPut()");
 83             sb.Append(Environment.NewLine);
 84             sb.Append(" {");
 85             sb.Append(Environment.NewLine);
 86             sb.Append(" return \\"Hello world!\\";");
 87             sb.Append(Environment.NewLine);
 88             sb.Append(" }");
 89             sb.Append(Environment.NewLine);
 90             sb.Append(" }");
 91             sb.Append(Environment.NewLine);
 92             sb.Append("}");
 93 
 94             string code = sb.ToString();
 95             Console.WriteLine(code);
 96             Console.WriteLine();
 97 
 98             return code;
 99         }
100 
101 
102         private static void Timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
103         {
104             System.Diagnostics.Process[] processes = System.Diagnostics.Process.GetProcessesByName("HelloWorld.exe");
105 
106             int maxProcessCount = 8;
107             // 如果超过了最大允许任务数据不再触发新的进程任务
108             if (processes.Length >= maxProcessCount)
109                 return;
110 
111             List<int> tasks = new List<int>();
112             Random random = new Random(1000);
113             for (int i = 0; i < maxProcessCount - processes.Length; i++)
114             {
115                 tasks.Add(random.Next(1000));
116             }
117 
118 
119             System.Threading.Tasks.Task[] taskItems = new System.Threading.Tasks.Task[tasks.Count];
120 
121             for (int i = 0; i < tasks.Count; i++)
122             {
123                 //TaskBase taskSubmit = new ParseMrToHdfsFileItemTask();
124                 //taskItems[i] = System.Threading.Tasks.Task.Factory.StartNew(taskSubmit.Submit, new ParseMrToHdfsFileItemTaskArgument() { Task = tasks[i], ComputeNode = this.ComputeNode }, TaskCreationOptions.PreferFairness);
125                 taskItems[i] = System.Threading.Tasks.Task.Factory.StartNew((object state) =>
126                 {
127                     System.Diagnostics.Process process = new System.Diagnostics.Process();
128                 //System.Diagnostics.Process process = System.Diagnostics.Process.Start("HelloWorld.exe", state.ToString());
129                 process.StartInfo = new System.Diagnostics.ProcessStartInfo();
130                     process.StartInfo.CreateNoWindow = true;
131                     process.StartInfo.UseShellExecute = false;
132                     process.StartInfo.WindowStyle = System.Diagnostics.ProcessWindowStyle.Hidden;
133                     process.StartInfo.Arguments = state.ToString();
134                     process.StartInfo.FileName = "HelloWorld.exe";
135                     process.Start();
136                     process.WaitForExit();
137                     process.Close();
138                     process.Dispose();
139                 }, tasks[i]);
140             }
141 
142             System.Threading.Tasks.Task.WaitAll(taskItems);
143         }
144 
145         private static void GenerateExe()
146         {
147             // 创建编译器对象
148             CSharpCodeProvider p = new CSharpCodeProvider();
149             ICodeCompiler cc = p.CreateCompiler();
150 
151             // 设置编译参数
152             CompilerParameters options = new CompilerParameters();
153             options.ReferencedAssemblies.Add("System.dll");
154             options.ReferencedAssemblies.Add("MutilProcessDemo.exe");
155             options.GenerateExecutable = true;
156             options.OutputAssembly = "HelloWorld.exe";
157 
158             //options.ReferencedAssemblies.Add("System.Windows.Forms.dll");
159             //options.EmbeddedResources.Add("Data.xml"); // 添加内置资源
160             //options.CompilerOptions += " /target:winexe";
161             //options.CompilerOptions += " /res:Resource1.res";
162             //options.CompilerOptions += " /win32icon:test.ico";
163 
164             // 创建源码
165 
166             // 1. 使用CodeDom创建源码
167             //CodeCompileUnit cu = new CodeCompileUnit();
168             //CodeNamespace Samples = new CodeNamespace("Samples");
169             //cu.Namespaces.Add(Samples);
170             //Samples.Imports.Add(new CodeNamespaceImport("System"));
171             //CodeTypeDeclaration Class1 = new CodeTypeDeclaration("Class1");
172             //Samples.Types.Add(Class1);
173             //CodeEntryPointMethod Start = new CodeEntryPointMethod();
174             //CodeMethodInvokeExpression cs1 = new CodeMethodInvokeExpression(
175             // new CodeTypeReferenceExpression("System.Console"), "WriteLine",
176             // new CodePrimitiveExpression("Hello World!")
177             // );
178             //Start.Statements.Add(new CodeExpressionStatement(cs1));
179             //Class1.Members.Add(Start);
180 
181             // 2. 直接指定源码字符串
182             string code = @"
183 using System;
184 using MutilProcessDemo;
185 
186 namespace Samples
187 {
188     public class Class1
189     {
190         static void Main(string[] args)
191         {
192             Console.WriteLine(""Hello, World!"");
193             MutilProcessDemo.Program.DoMethod(args);
194             Console.WriteLine(DateTime.Now.ToString());
195         }
196     }
197 }";
198             CodeSnippetCompileUnit codeSnippetCompileUnit = new CodeSnippetCompileUnit(code);
199 
200             // 开始编译
201             CompilerResults compilerResults = cc.CompileAssemblyFromDom(options, codeSnippetCompileUnit);
202 
203             // 显示编译信息
204             if (compilerResults.Errors.Count == 0)
205                 Console.WriteLine("{0}compiled ok!", compilerResults.CompiledAssembly.Location);
206             else
207             {
208                 Console.WriteLine("Complie Error:");
209                 foreach (CompilerError error in compilerResults.Errors)
210                     Console.WriteLine(" {0}", error);
211             }
212         }
213 
214         public static void DoMethod(string[] args)
215         {
216             System.Console.WriteLine("begin ..." + args[0]);
217 
218             for (int i = 0; i < int.Parse(args[0]); i++)
219             {
220                 System.Threading.Thread.Sleep(20);
221             }
222 
223             System.Console.WriteLine("end..." + args[0]);
224         }
225     }
226 }

上边的程序运行之后会在\\MutilProcessDemo\\bin\\Debug\\下出现两个可执行的.exe：HelloWorld.exe、MutilProcessDemo.exe。

参考文章：

c#动态编译，程序集的动态创建（http://www.360doc.com/content/14/1014/11/5054188_416763069.shtml）