P/Invoke 编组和解组 C# 和非托管 DLL 之间的二维数组、结构和指针
Posted
技术标签:
【中文标题】P/Invoke 编组和解组 C# 和非托管 DLL 之间的二维数组、结构和指针【英文标题】：P/Invoke marshaling and unmarshalling 2D array, structure and pointers between C# and unmanaged DLL 【发布时间】：2017-10-30 02:21:22 【问题描述】：Flann C++ library 具有 C、C++、Python、Matlab 和 Ruby 的包装器，但没有可用的 C# 包装器。我正在尝试围绕从here 下载的flann.dll 32 位非托管DLL 创建一个C# 包装器。
作为 PInvoke/编组的新手，我很确定我没有正确执行对 DLL 的 C# P/Invoke 调用。我基本上是在尝试在 C# 中镜像可用的Python wrapper。主要的混淆领域是：
我不确定如何在参数类型为float* 的 C# 中的二维托管矩形数组之间编组（输入）和解组（输出），即指向存储在 row major order 中的查询集的指针（根据 cmets in flann.h)。
我也不确定我如何传递对 C 的结构引用是否正确，即struct FLANNParameters*
IntPtr 是否适合引用 typedef void* 和 int* indices？
非托管 C（flann.dll 库）
我需要使用的从flann.h 导出的公共C++ 方法如下：
typedef void* FLANN_INDEX; /* deprecated */
typedef void* flann_index_t;
FLANN_EXPORT extern struct FLANNParameters DEFAULT_FLANN_PARAMETERS;
// dataset = pointer to a query set stored in row major order
FLANN_EXPORT flann_index_t flann_build_index(float* dataset,
int rows,
int cols,
float* speedup,
struct FLANNParameters* flann_params);
FLANN_EXPORT int flann_free_index(flann_index_t index_id,
struct FLANNParameters* flann_params);
FLANN_EXPORT int flann_find_nearest_neighbors(float* dataset,
int rows,
int cols,
float* testset,
int trows,
int* indices,
float* dists,
int nn,
struct FLANNParameters* flann_params);
托管 C# 包装器（我的实现）
这是我基于上述公开方法的 C# 包装器。
NativeMethods.cs
using System;
using System.Runtime.InteropServices;
namespace FlannWrapper
/// <summary>
/// Methods to map between native unmanaged C++ DLL and managed C#
/// Trying to mirror: https://github.com/mariusmuja/flann/blob/master/src/cpp/flann/flann.h
/// </summary>
public class NativeMethods
/// <summary>
/// 32-bit flann dll obtained from from http://sourceforge.net/projects/pointclouds/files/dependencies/flann-1.7.1-vs2010-x86.exe/download
/// </summary>
public const string DllWin32 = @"C:\Program Files (x86)\flann\bin\flann.dll";
/// <summary>
/// C++: flann_index_t flann_build_index(float* dataset, int rows, int cols, float* speedup, FLANNParameters* flann_params)
/// </summary>
[DllImport(DllWin32, EntryPoint = "flann_build_index", CallingConvention = CallingConvention.Cdecl, SetLastError = true)]
public static extern IntPtr flannBuildIndex([In] [MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.R4)] float[,] dataset, // ??? [In] IntPtr dataset ???
int rows, int cols,
ref float speedup, // ???
[In] ref FlannParameters flannParams); // ???
/// <summary>
/// C++: int flann_free_index(flann_index_t index_ptr, FLANNParameters* flann_params)
/// </summary>
[DllImport(DllWin32, EntryPoint = "flann_free_index", CallingConvention = CallingConvention.Cdecl, SetLastError = true)]
public static extern int flannFreeIndex(IntPtr indexPtr, // ???
[In] ref FlannParameters flannParams); // ??? [In, MarshalAs(UnmanagedType.LPStruct)] FlannParameters flannParams);
/// <summary>
/// C++: int flann_find_nearest_neighbors_index(flann_index_t index_ptr, float* testset, int tcount, int* result, float* dists, int nn, FLANNParameters* flann_params)
/// </summary>
[DllImport(DllWin32, EntryPoint = "flann_find_nearest_neighbors_index", CallingConvention = CallingConvention.Cdecl, SetLastError = true)]
public static extern int flannFindNearestNeighborsIndex(IntPtr indexPtr, // ???
[In] [MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.R4)] float[,] testset, // ??? [In] IntPtr dataset ???
int tCount,
[Out] IntPtr result, // ??? [Out] [MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.R4)] int[,] result,
[Out] IntPtr dists, // ???
int nn,
[In] ref FlannParameters flannParams); // ???
FlannTest.cs
using System;
using System.IO;
using System.Runtime.InteropServices;
using Microsoft.VisualStudio.TestTools.UnitTesting;
namespace FlannWrapper
[TestClass]
public class FlannTest : IDisposable
private IntPtr curIndex;
protected FlannParameters flannParams;
// protected GCHandle gcHandle;
[TestInitialize]
public void TestInitialize()
this.curIndex = IntPtr.Zero;
// Initialise Flann Parameters
this.flannParams = new FlannParameters(); // use defaults
this.flannParams.algorithm = FlannAlgorithmEnum.FLANN_INDEX_KDTREE;
this.flannParams.trees = 8;
this.flannParams.logLevel = FlannLogLevelEnum.FLANN_LOG_WARN;
this.flannParams.checks = 64;
[TestMethod]
public void FlannNativeMethodsTestSimple()
int rows = 3, cols = 5;
int tCount = 2, nn = 3;
float[,] dataset2D = 1.0f, 1.0f, 1.0f, 2.0f, 3.0f,
10.0f, 10.0f, 10.0f, 3.0f, 2.0f,
100.0f, 100.0f, 2.0f, 30.0f, 1.0f ;
//IntPtr dtaasetPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(float)) * dataset2D.Length);
float[,] testset2D = 1.0f, 1.0f, 1.0f, 1.0f, 1.0f,
90.0f, 90.0f, 10.0f, 10.0f, 1.0f ;
//IntPtr testsetPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(float)) * testset2D.Length);
int outBufferSize = tCount * nn;
int[] result = new int[outBufferSize];
int[,] result2D = new int[tCount, nn];
IntPtr resultPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(int)) * result.Length);
float[] dists = new float[outBufferSize];
float[,] dists2D = new float[tCount, nn];
IntPtr distsPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(float)) * dists.Length);
try
// Copy the array to unmanaged memory.
//Marshal.Copy(testset, 0, testsetPtr, testset.Length);
//Marshal.Copy(dataset, 0, datasetPtr, dataset.Length);
if (this.curIndex != IntPtr.Zero)
// n - number of bytes which is enough to keep any type used by function
NativeMethods.flannFreeIndex(this.curIndex, ref this.flannParams);
this.curIndex = IntPtr.Zero;
//GC.KeepAlive(this.curIndex); // TODO
float speedup = 0.0f; // TODO: ctype float
Console.WriteLine("Computing index.");
this.curIndex = NativeMethods.flannBuildIndex(dataset2D, rows, cols, ref speedup, ref this.flannParams);
NativeMethods.flannFindNearestNeighborsIndex(this.curIndex, testset2D, tCount, resultPtr, distsPtr, nn, ref this.flannParams);
// Copy unmanaged memory to managed arrays.
Marshal.Copy(resultPtr, result, 0, result.Length);
Marshal.Copy(distsPtr, dists, 0, dists.Length);
// Clutching straws, convert 1D to 2D??
for(int row=0; row<tCount; row++)
for(int col=0; col<nn; col++)
int buffIndex = row*nn + col;
result2D[row, col] = result[buffIndex];
dists2D[row, col] = dists[buffIndex];
finally
// Free unmanaged memory -- [BREAKPOINT HERE]
// Free input pointers
//Marshal.FreeHGlobal(testsetPtr);
//Marshal.FreeHGlobal(datasetPtr);
// Free output pointers
Marshal.FreeHGlobal(resultPtr);
Marshal.FreeHGlobal(distsPtr);
[TestCleanup]
public void TestCleanup()
if (this.curIndex != IntPtr.Zero)
NativeMethods.flannFreeIndex(this.curIndex, ref flannParams);
Marshal.FreeHGlobal(this.curIndex);
this.curIndex = IntPtr.Zero;
// gcHandle.Free();
FlannParams.cs
尝试镜像Python FLANNParameters class 和C struct FLANNParameters。
using System;
using System.Runtime.InteropServices;
namespace FlannWrapper
// FieldOffsets set based on assumption that C++ equivalent of int, uint, float, enum are all 4 bytes for 32-bit
[StructLayout(LayoutKind.Explicit)]
public class FLANNParameters
[FieldOffset(0)]
public FlannAlgorithmEnum algorithm;
[FieldOffset(4)]
public int checks;
[FieldOffset(8)]
public float eps;
[FieldOffset(12)]
public int sorted;
[FieldOffset(16)]
public int maxNeighbors;
[FieldOffset(20)]
public int cores;
[FieldOffset(24)]
public int trees;
[FieldOffset(28)]
public int leafMaxSize;
[FieldOffset(32)]
public int branching;
[FieldOffset(36)]
public int iterations;
[FieldOffset(40)]
public FlannCentersInitEnum centersInit;
[FieldOffset(44)]
public float cbIndex;
[FieldOffset(48)]
public float targetPrecision;
[FieldOffset(52)]
public float buildWeight;
[FieldOffset(56)]
public float memoryWeight;
[FieldOffset(60)]
public float sampleFraction;
[FieldOffset(64)]
public int tableNumber;
[FieldOffset(68)]
public int keySize;
[FieldOffset(72)]
public int multiProbeLevel;
[FieldOffset(76)]
public FlannLogLevelEnum logLevel;
[FieldOffset(80)]
public long randomSeed;
/// <summary>
/// Default Constructor
/// Ref https://github.com/mariusmuja/flann/blob/master/src/python/pyflann/flann_ctypes.py : _defaults
/// </summary>
public FlannParameters()
this.algorithm = FlannAlgorithmEnum.FLANN_INDEX_KDTREE;
this.checks = 32;
this.eps = 0.0f;
this.sorted = 1;
this.maxNeighbors = -1;
this.cores = 0;
this.trees = 1;
this.leafMaxSize = 4;
this.branching = 32;
this.iterations = 5;
this.centersInit = FlannCentersInitEnum.FLANN_CENTERS_RANDOM;
this.cbIndex = 0.5f;
this.targetPrecision = 0.9f;
this.buildWeight = 0.01f;
this.memoryWeight = 0.0f;
this.sampleFraction = 0.1f;
this.tableNumber = 12;
this.keySize = 20;
this.multiProbeLevel = 2;
this.logLevel = FlannLogLevelEnum.FLANN_LOG_WARN;
this.randomSeed = -1;
public enum FlannAlgorithmEnum : int
FLANN_INDEX_KDTREE = 1
public enum FlannCentersInitEnum : int
FLANN_CENTERS_RANDOM = 0
public enum FlannLogLevelEnum : int
FLANN_LOG_WARN = 3
不正确的输出 - 调试模式，立即窗口
?result2D
int[2, 3]
[0, 0]: 7078010
[0, 1]: 137560165
[0, 2]: 3014708
[1, 0]: 3014704
[1, 1]: 3014704
[1, 2]: 48
?dists2D
float[2, 3]
[0, 0]: 2.606415E-43
[0, 1]: 6.06669328E-34
[0, 2]: 9.275506E-39
[1, 0]: 1.05612418E-38
[1, 1]: 1.01938872E-38
[1, 2]: 1.541428E-43
如您所见，在调试模式下运行测试时我没有收到任何错误，但我知道输出肯定是不正确的 - 由于内存寻址不当导致的垃圾值。我还包括了我尝试过但没有成功的替代编组签名（请参阅带有???的 cmets）。
Ground Truth Python（调用 PyFlann 库）
为了找出正确的结果，我使用可用的 Python 库 - PyFlann 实现了一个快速测试。
FlannTest.py
import pyflann
import numpy as np
dataset = np.array(
[[1., 1., 1., 2., 3.],
[10., 10., 10., 3., 2.],
[100., 100., 2., 30., 1.] ])
testset = np.array(
[[1., 1., 1., 1., 1.],
[90., 90., 10., 10., 1.] ])
flann = pyflann.FLANN()
result, dists = flann.nn(dataset, testset, num_neighbors = 3,
algorithm="kdtree", trees=8, checks=64) # flann parameters
# Output
print("\nResult:")
print(result)
print("\nDists:")
print(dists)
在底层，PyFlann.nn() 调用公开的 C 方法，我们可以通过查看 index.py 看出这一点。
正确输出
Result:
[[0 1 2]
[2 1 0]]
Dists:
[[ 5.00000000e+00 2.48000000e+02 2.04440000e+04]
[ 6.64000000e+02 1.28500000e+04 1.59910000e+04]]
任何有关正确方法的帮助将不胜感激。谢谢。
【问题讨论】：
你的代码有很多问题。因为有这么多，我无法面对这个问题，因为回答将意味着为你完成整个工作。如果你能把它归结为更小的块，那就更好了。就目前而言，这里没有什么是以前无数问题中没有处理过的，但是将所有问题放在一起会使这个问题只与您相关。因此，它应该被关闭。 【参考方案1】：当您使用 p/invoke 时，您必须停止考虑“托管”，而应考虑物理二进制布局、32 位与 64 位等。此外，当被调用的本机二进制始终在进程内运行时（例如在这里，但使用 COM 服务器可能会有所不同）它比进程外更容易，因为您不必考虑太多关于编组/序列化、引用与输出等问题。
另外，您不需要告诉 .NET 它已经知道什么。浮点数组是 R4 的 LPArray，您不必指定它。越简单越好。
所以，首先flann_index_t。它在 C 中被定义为void *，所以它肯定是IntPtr（“某物”上的不透明指针）。
然后，结构。在 C 中作为简单指针传递的结构可以在 C# 中作为 ref 参数传递，如果您将其定义为 struct。如果您将其定义为class，请不要使用ref。一般来说，我更喜欢将 struct 用于 C 结构。
您必须确保结构定义明确。通常，您使用 LayoutKind.Sequential，因为 .NET p/invoke 将像 C 编译器一样打包参数。所以你不必使用显式，特别是当参数是标准的（而不是小东西）时，比如 int，float，所以你可以删除所有 FieldOffset 并使用 LayoutKind.Sequential 如果所有成员都被正确声明......但这不是案例。
对于类型，就像我说的那样，你真的必须考虑二进制并问自己你使用的每种类型，它的二进制布局，大小是多少？ int 是（使用 99.9% C 编译器）32 位。 float 和 double 是 IEEE 标准，因此不应该有关于它们的问题。枚举通常基于int，但这可能会有所不同（在 C 和 .NET 中，为了能够匹配 C）。 long 是（使用 99.0% C 编译器）32 位，不是 64 位。所以 .NET 等价物是 Int32 (int)，而不是 Int64 (long)。
因此，您应该更正您的FlannParameters 结构并将long 替换为int。如果您真的想确定一个给定的结构，请使用与用于编译您正在调用的库相同的 C 编译器检查 Marshal.SizeOf(mystruct) 和 C 的 sizeof(mystruct)。他们应该是一样的。如果不是，则 .NET 定义中存在错误（包装、成员大小、顺序等）。
这里是修改后的定义和调用代码，似乎可以工作。
static void Main(string[] args)
int rows = 3, cols = 5;
int tCount = 2, nn = 3;
float[,] dataset2D = 1.0f, 1.0f, 1.0f, 2.0f, 3.0f,
10.0f, 10.0f, 10.0f, 3.0f, 2.0f,
100.0f, 100.0f, 2.0f, 30.0f, 1.0f ;
float[,] testset2D = 1.0f, 1.0f, 1.0f, 1.0f, 1.0f,
90.0f, 90.0f, 10.0f, 10.0f, 1.0f ;
var fparams = new FlannParameters();
var index = NativeMethods.flannBuildIndex(dataset2D, rows, cols, out float speedup, ref fparams);
var indices = new int[tCount, nn];
var idists = new float[tCount, nn];
NativeMethods.flannFindNearestNeighborsIndex(index, testset2D, tCount, indices, idists, nn, ref fparams);
NativeMethods.flannFreeIndex(index, ref fparams);
[DllImport(DllWin32, EntryPoint = "flann_build_index", CallingConvention = CallingConvention.Cdecl)]
public static extern IntPtr flannBuildIndex(float[,] dataset,
int rows, int cols,
out float speedup, // out because, it's and output parameter, but ref is not a problem
ref FlannParameters flannParams);
[DllImport(DllWin32, EntryPoint = "flann_free_index", CallingConvention = CallingConvention.Cdecl)]
public static extern int flannFreeIndex(IntPtr indexPtr, ref FlannParameters flannParams);
[DllImport(DllWin32, EntryPoint = "flann_find_nearest_neighbors_index", CallingConvention = CallingConvention.Cdecl, SetLastError = true)]
public static extern int flannFindNearestNeighborsIndex(IntPtr indexPtr,
float[,] testset,
int tCount,
[In, Out] int[,] result, // out because it may be changed by C side
[In, Out] float[,] dists,// out because it may be changed by C side
int nn,
ref FlannParameters flannParams);
[StructLayout(LayoutKind.Sequential)]
public struct FlannParameters
public FlannAlgorithmEnum algorithm;
public int checks;
public float eps;
public int sorted;
public int maxNeighbors;
public int cores;
public int trees;
public int leafMaxSize;
public int branching;
public int iterations;
public FlannCentersInitEnum centersInit;
public float cbIndex;
public float targetPrecision;
public float buildWeight;
public float memoryWeight;
public float sampleFraction;
public int tableNumber;
public int keySize;
public int multiProbeLevel;
public FlannLogLevelEnum logLevel;
public int randomSeed;
注意：我尝试过使用具体的flann参数值，但是在这种情况下库崩溃了，我不知道为什么......
【讨论】：
非常感谢您的清晰解释。这有助于我更好地理解我对编组过程所做的许多微妙的误解。没错，当指定自定义 Flann 参数时，库会崩溃。我试图找出原因。如果有，我会尽快更新。以上是关于P/Invoke 编组和解组 C# 和非托管 DLL 之间的二维数组、结构和指针的主要内容，如果未能解决你的问题，请参考以下文章
通过 P/Invoke 将 MFC CArray 编组为 C#