写一个程序，模拟人操作鼠标，进行操作，包括点击，和使用应用程序（如：浏览器），求实现方法。

Posted 2023-04-24

用什么语言写？中间 应该 还牵扯到了 桌面坐标的问题。。 还有应用程序中，需要点击信息的 坐标等。。。主要是 浏览器中 所需 点击 信息的 坐标

你可以试试看用华华鼠标自动点击器呀 ，这个软件是模拟鼠标点击的，支持多坐标点击的。完全能实现你要求的这些功能，可以在百度搜索 华华鼠标自动点击器 进入官网下载 参考技术A 用什么语言写？中间 应该 还牵扯到了 桌面坐标的问题。。 参考技术B 我知道，这个不会编程的人都可以“这个软件的名字叫做按键精灵”欢迎采纳。不懂追问我

在VR中模拟用鼠标操作电脑并实现简单画图的小程序

 

一、概述

1.实现的基本操作是：

　　1）用手柄抓住黄色的方块代表手抓住鼠标。

　　2）通过移动手柄模拟鼠标移动，电脑屏幕上的光标跟着移动。

　　3）当光标移动到一个Button上时，Button高亮，离开时Button取消高亮，点击Button触发点击事件。

　　4）当点击Button之后，打开一个画图程序，可以用光标在颜色选择区选择一种颜色，然后在画图区根据光标的移动轨迹，画出选择颜色的光标移动路径的曲线；

2.脚本

　　1）ComputerController挂在代表电脑的Canvas上，本例挂在Computer上；

　　2）MouseController挂在一个代表鼠标的物体上，本例挂在Mouse上；

　　3）ComputerCursorController 挂在表示光标的一个Image上，本例挂在Cursor上；

　　4)  ComputerClickable挂在所有可点击的应用程序图标上，在本例中只有一个应用程序，挂在PaintProgramIcon上；

　　5）PaintProgram一个画图程序，在本例中挂在PaintProgram这个Panel上

3.场景的Hierachy面板

 二、实现

1.手柄的操作的鼠标设置：

由于VRTK这个插件集成了很好的物理交互功能，所以就手柄与场景物体交互方面选择用VRTK这个插件。

下面是代表鼠标的黄色的Cube的设置

首先需要挂上如上面图中的所有组件：

　　1）Rigidbody需要设置约束：禁止Y方向的移动，以及任意方向的旋转；

　　2）将VRTK_TrackObjectGrabAttach这个脚本拖到VRTK_InteractableObject的Grab Attach Mechanic上；

　　3）将Grab Override Button选择一个手柄上不存在键，HTC VIVE中这个键是Button One，我们将在代码中设置抓取；

　　4）将VRTK_InteractableObject这个组件上的IsGrabable和IsUsable打上勾；

　　5）将VRTK_Interact Controller Apperance的Hide Controller On Grab打勾；

Mouse Controller这个脚本就是控制鼠标移动的；

using System;
using UnityEngine;
using VRTK;

[RequireComponent(typeof(VRTK_InteractableObject))]
public class MouseController : MonoBehaviour
{
    public Transform mousePad;//鼠标垫

    public Action ClickDown;//当手柄上的use键按下的时候，引发的事件（Trigger键）；
    public Action ClickUp;//当手柄上的use键抬起的时候，引发的事件；

    private Rect mappingRect;//这个鼠标的移动范围（用来映射电脑屏幕上的光标的位置)
    private Vector2 mouseReletiveToMousePadPostion;//鼠标相对于鼠标垫的位置

    Rigidbody rig;
    BoxCollider boxCollider;
    VRTK_InteractableObject mouse;
    VRTK_InteractGrab controller;//当前正在使用鼠标的手柄

    bool isUsedToGrab;//鼠标是不是被手柄抓住
    float releaseDistance;//手柄离开鼠标的距离（不再抓住鼠标了）

    void Start()
    {
        //初始化一些数据 
        mappingRect = GetMatchRect(mousePad);
        CalculateMouseReletivePos();
        mouse = GetComponent<VRTK_InteractableObject>();
        rig = GetComponent<Rigidbody>();
        boxCollider = GetComponent<BoxCollider>();
        releaseDistance = boxCollider.bounds.size.y * 2f;

        //监听手柄触碰到鼠标的事件
        mouse.InteractableObjectTouched += Mouse_InteractableObjectTouched;
        //监听手柄不再触碰鼠标事件
        mouse.InteractableObjectUntouched += Mouse_InteractableObjectUntouched;
    }

    /// <summary>
    ///  //当手柄触碰到鼠标时，执行的事件
    /// </summary>
    /// <param name="sender"></param>
    /// <param name="e"></param>
    private void Mouse_InteractableObjectTouched(object sender, InteractableObjectEventArgs e)
    {
        //监听手柄使用键按下时的事件
        mouse.InteractableObjectUsed += Mouse_InteractableObjectUsed;
    }

    /// <summary>
    /// 当手柄use键按下时，执行的事件
    /// </summary>
    /// <param name="sender"></param>
    /// <param name="e"></param>
    private void Mouse_InteractableObjectUsed(object sender, InteractableObjectEventArgs e)
    {
        isUsedToGrab = true;//设置抓取住了鼠标
        controller = e.interactingObject.GetComponent<VRTK_InteractGrab>();//设置当前抓取鼠标的手柄
        controller.AttemptGrab();//强制手柄抓住鼠标
        mouse.InteractableObjectUsed -= Mouse_InteractableObjectUsed;//不再监听use键按下事件
        //监听当前操作的手柄use键按下时的事件
        controller.GetComponent<VRTK_InteractUse>().UseButtonPressed += MouseController_UseButtonPressed;
        //监听当前操作的手柄use键抬起时的事件
        controller.GetComponent<VRTK_InteractUse>().UseButtonReleased += MouseController_UseButtonReleased;
    }

    private void MouseController_UseButtonReleased(object sender, ControllerInteractionEventArgs e)
    {
        //引发事件
        if (ClickUp != null)
        {
            ClickUp();
        }
    }

    private void MouseController_UseButtonPressed(object sender, ControllerInteractionEventArgs e)
    {
        if (ClickDown != null)
        {
            ClickDown();
        }
    }

    private void Mouse_InteractableObjectUntouched(object sender, InteractableObjectEventArgs e)
    {
        rig.velocity = Vector3.zero;//一旦手柄不再和鼠标有碰撞，这时要让鼠标的速度为0
        if (isUsedToGrab)//如果之前是抓住了鼠标的
        {
            isUsedToGrab = false;
            controller.GetComponent<VRTK_InteractUse>().UseButtonPressed -= MouseController_UseButtonPressed;
            controller.GetComponent<VRTK_InteractUse>().UseButtonReleased -= MouseController_UseButtonReleased;
            controller.ForceRelease();//强制松开
        }
        else
        {
            mouse.InteractableObjectUsed -= Mouse_InteractableObjectUsed;
        }
        controller = null;
    }

    void Update()
    {
        if (isUsedToGrab)//只有当鼠标被抓住时才执行
        {
            //根据鼠标移动的速度，手柄相应程度的振动
            VRTK_ControllerReference controllerReference = VRTK_ControllerReference.GetControllerReference(controller.gameObject);
            float force = VRTK_SDK_Bridge.GetControllerVelocity(controllerReference).sqrMagnitude;
            VRTK_SDK_Bridge.HapticPulse(controllerReference, force / 3);

            CalculateMouseReletivePos();
            //判定当前手柄是不是离开了鼠标
            if ((transform.position - controller.transform.position).sqrMagnitude > releaseDistance * releaseDistance)
            {
                controller.ForceRelease();//强制松开
            }
        }
    }

    public Vector2 MouseReletiveToTablePos
    {
        get
        {
            return mouseReletiveToMousePadPostion;
        }
    }

    /// <summary>
    /// 计算鼠标相对于鼠标垫的位置，x和y都是0-1 ；
    /// </summary>
    void CalculateMouseReletivePos()
    {
        float x = Mathf.InverseLerp(mappingRect.xMin, mappingRect.xMax, transform.position.x);
        float y = Mathf.InverseLerp(mappingRect.yMin, mappingRect.yMax, transform.position.z);
        mouseReletiveToMousePadPostion.Set(x, y);
    }

    /// <summary>
    /// 设置鼠标的移动范围
    /// </summary>
    /// <param name="content"></param>
    /// <returns></returns>
    public Rect GetMatchRect(Transform content)
    {
        Vector3 contentSize = content.GetComponent<MeshRenderer>().bounds.size;
        Vector3 selfSize = GetComponent<MeshRenderer>().bounds.size;
        //让Rect的position是在鼠标垫的左下角
        Vector2 pos = new Vector2(content.localPosition.x - contentSize.x * 0.5f + selfSize.x * 0.5f, content.localPosition.z - contentSize.z * 0.5f + selfSize.z * 0.5f);
        //设置Rect的长度和宽度（应该减去鼠标自身的长宽）
        Vector2 size = new Vector2(contentSize.x - selfSize.x, contentSize.z - selfSize.z);
        Rect rect = new Rect(pos, size);
        return rect;
    }

}

2.电脑屏幕光标的设置：

需要注意的是：锚点在左下角，pivot在自身矩形的左上角(鼠标的尖端的位置)

ComputerCursorController是实现鼠标和光标位置映射的类

using System;
using UnityEngine;

public class ComputerCursorController : MonoBehaviour
{
    public MouseController mouseController;

    public Action OnMoved;
    public Action ClickedDown;
    public Action ClickedUp;

    private Rect rect;//表示电脑屏幕范围的Rect
    private RectTransform rectTransform;//这个光标的RectTransform

    void Start()
    {
        //代表电脑屏幕范围的Rect的位置计算
        rect = transform.parent.GetComponent<RectTransform>().rect;
        rect.position += new Vector2(rect.width / 2f, rect.height / 2f);
        rectTransform = transform as RectTransform;
    }

    void OnEnable()
    {
        mouseController.ClickUp += MouseController_ClickUp);
        mouseController.ClickDown += MouseController_ClickDown;
    }

    void OnDisable()
    {
        mouseController.ClickUp -= MouseController_ClickUp;
        mouseController.ClickDown -= MouseController_ClickDown;
    }

    private void MouseController_ClickDown()
    {
        if (ClickedDown != null)
        {
            ClickedDown();
        }
    }

    private void MouseController_ClickUp()
    {
        if (ClickedUp != null)
        {
            ClickedUp();
        }
    }

    void Update()
    {
        Vector2 parameter = mouseController.MouseReletiveToTablePos;
        Vector2 vector = new Vector2(Mathf.Lerp(rect.xMin, rect.xMax, parameter.x), Mathf.Lerp(rect.yMin, rect.yMax, parameter.y));

        //当鼠标映射的位置和光标的位置不等的时候，说明这个时候鼠标是在移动的
        if (rectTransform.anchoredPosition != vector && OnMoved != null)
        {
            OnMoved();
        }
        rectTransform.anchoredPosition = vector;
    }
}

3.响应光标的点击事件

在电脑中点击桌面上的一个图标的时候，是可以进入应用程序的，在VR中直接单击进入程序就好，ComputerController这个类用来控制光标引发的一些事件

using System;
using UnityEngine;

public class ComputerController : MonoBehaviour
{
    public ComputerClickable[] clickables;//桌面上所有可点击的应用程序图标
    public ComputerCursorController cursorController;//光标

    private ComputerClickable currentClickable;//当前光标所在图标
    private ComputerClickable cacheClickedClickable;//缓存的图标

    [SerializeField]
    private Canvas canvas;//代表Computer的画布

    public Action Clicked;
    public Action UnClicked;

    void OnEnable()
    {
        cursorController.OnMoved += CheckClickablesIsHoverByCursor;
        cursorController.ClickedDown += ClickDown;
        cursorController.ClickedUp += ClickUp;
    }


    void OnDisable()
    {
        cursorController.OnMoved -= CheckClickablesIsHoverByCursor;
        cursorController.ClickedDown -= ClickDown;
        cursorController.ClickedUp -= ClickUp;
    }

    /// <summary>
    /// 检查光标是不是移动到应用程序图标上
    /// </summary>
    private void CheckClickablesIsHoverByCursor()
    {
        for (int i = 0; i < clickables.Length; i++)
        {
            if (clickables[i].CheckHoverByCursor(CursorPosition))
            {
                currentClickable = clickables[i];
                return;
            }
        }
        currentClickable = null;
    }

    private void ClickDown()
    {
        if (currentClickable != null)
        {
            currentClickable.Click();
            cacheClickedClickable = currentClickable;
            currentClickable = null;
        }
        if (Clicked != null)
        {
            Clicked();
        }
    }

    private void ClickUp()
    {
        if (cacheClickedClickable != null)
        {
            cacheClickedClickable.UnClick();
            cacheClickedClickable = null;
        }
        if (UnClicked != null)
        {
            UnClicked();
        }
    }

    public Canvas Canvas
    {
        get
        {
            return canvas;
        }
    }

    /// <summary>
    /// 光标的位置
    /// </summary>
    public Vector2 CursorPosition
    {
        get
        {
            return RectTransformUtility.WorldToScreenPoint(canvas.worldCamera, cursorController.transform.position); 
        }
    }
}

4.可点击的应用程序图标

using UnityEngine;
using UnityEngine.UI;
using UnityEngine.EventSystems;

public class ComputerClickable : MonoBehaviour
{
    RectTransform rectTransform;
    Canvas canvas;
    Button button;
    bool isHighlighter;//当前Button是否高亮

    void Start()
    {
        //初始化字段 
        rectTransform = transform as RectTransform;
        canvas = GetComponentInParent<Canvas>();
        button = GetComponent<Button>();
        button.onClick.AddListener(() => Debug.Log("Clicked"));
    }

    /// <summary>
    /// 光标是否移动到自身上
    /// </summary>
    /// <param name="cursorPos">光标的位置</param>
    /// <returns>True 当前光标在自身上</returns>
    public bool CheckHoverByCursor(Vector2 cursorPos)
    {
        //检查一个RectTransform是不是包含一个点
        bool isHorver = RectTransformUtility.RectangleContainsScreenPoint(rectTransform, cursorPos, canvas.worldCamera);
        PointerEventData eventData = new PointerEventData(EventSystem.current);
        if (isHorver && !isHighlighter)//如果包含，且当前Button没有高亮
        {
            //引发Button高亮
            ExecuteEvents.Execute(gameObject, eventData, ExecuteEvents.pointerEnterHandler);      
            isHighlighter = true;
        }
        else if (!isHorver && isHighlighter)//如果没有包含，但是Button高亮，说明光标已经离开
        {
            isHighlighter = false;
            ExecuteEvents.Execute(gameObject, eventData, ExecuteEvents.pointerUpHandler);
            ExecuteEvents.Execute(gameObject, eventData, ExecuteEvents.pointerExitHandler);
            ExecuteEvents.Execute(gameObject, eventData, ExecuteEvents.deselectHandler);
        }
        return isHorver;
    }

    public void Click()
    {
        ExecuteEvents.Execute(gameObject, new PointerEventData(EventSystem.current), ExecuteEvents.pointerDownHandler);  
        button.onClick.Invoke();
    }

    public void UnClick()
    {
        ExecuteEvents.Execute(gameObject, new PointerEventData(EventSystem.current), ExecuteEvents.pointerUpHandler);
       // button.OnPointerUp(new PointerEventData(EventSystem.current));
    }
}

至此基本功能已经实现，接下来可以写一个小程序来试试；

三、简单的画图小程序 

画图小程序主要包含3个部分

　　1）颜色选择区（ColorPick）

　　2）颜色选择展示区(SelectColor)

　　3）画图区(PaintImage)

 

 

脚本功能实现：

using UnityEngine.UI;
using UnityEngine;
using System;
using System.Linq;
public class PaintProgram : MonoBehaviour
{
    public ComputerController computer;

    public int pictureWidth;//画图区的宽度
    public int pictureHeight;//画图区的高度
    public RawImage pictureImage;//画图区
    public RawImage colorPickImage;//颜色选择区
    public Image selectedColorDisplay;//颜色选择展示区

    private Texture2D pictureTex;//赋值给画图区的Texture
    private Texture2D colorPickTex;//颜色选择区的Texture
    private Rect pictureRect;//画图区区域
    private Rect colorPickerRect;//颜色选择区区域
 
    private bool canOperate;//当前是否可以操作（画图或者选择颜色）

    private bool isInDrawArea;//光标是否在画图区
    private bool isInPickArea;//光标是否在颜色选择区域
    private Color selectedColor = Color.red;//当前从颜色选择区选择的颜色 

    //在本例中是给每个像素赋值，用这两个参数，可以给一个区域内的像素赋值
    private Color[] c;//色块的颜色 
    private Vector2 lineSize;//画图区线的大小


    void Awake()
    {
        pictureTex = new Texture2D(pictureWidth, pictureHeight);
        pictureTex.filterMode = FilterMode.Point;
        pictureTex.wrapMode = TextureWrapMode.Clamp;
        pictureImage.texture = pictureTex;
        ResetPixes(Color.white);

        //设置画图区域矩形的位置
        pictureRect = pictureImage.GetComponent<RectTransform>().rect;
        pictureRect.position = computer.Canvas.transform.InverseTransformPoint(pictureImage.transform.position);
        pictureRect.center = pictureRect.position;
        //设置颜色选择区域矩形的位置
        colorPickerRect = colorPickImage.GetComponent<RectTransform>().rect;
        colorPickerRect.position = computer.Canvas.transform.InverseTransformPoint(colorPickImage.transform.position);
        colorPickerRect.center = colorPickerRect.position;

        colorPickTex = colorPickImage.texture as Texture2D;

    }

    void Start()
    {
        lineSize = 5 * new Vector2(pictureRect.width / (float)pictureWidth, pictureRect.height / (float)pictureHeight);
        c = new Color[(int)lineSize.x * (int)lineSize.y];
        c = Enumerable.Repeat<Color>(selectedColor, c.Length).ToArray<Color>();
    }

    void OnEnable()
    {
        computer.Clicked += OnMouseClick;
        computer.UnClicked += OnMouseUnClick;
    }

    void OnDisable()
    {
        computer.Clicked -= OnMouseClick;
        computer.UnClicked -= OnMouseUnClick;
    }

    void Update()
    {
        Vector2 relativeDrawPosition = GetRelativePosition(pictureRect);
        Vector2 relativePickPosition = GetRelativePosition(colorPickerRect);
        isInDrawArea = relativeDrawPosition.x >= 0f && relativeDrawPosition.x < 1f && relativeDrawPosition.y >= 0f && relativeDrawPosition.y < 1f;
        isInPickArea = relativePickPosition.x >= 0f && relativePickPosition.x < 1f && relativePickPosition.y >= 0f && relativePickPosition.y < 1f;
        if (isInDrawArea)//如果在画图区
        {
            if (canOperate)//鼠标点击了
            {
                SetPixel(relativeDrawPosition.x, relativeDrawPosition.y);
            }
        }
        else if (isInPickArea)//如果在颜色选择区
        {
            if (canOperate)//鼠标点击了
            {
                PickColor(relativePickPosition.x, relativePickPosition.y);
            }
        }
        else
        {
            if (canOperate)
            {
                canOperate = false;
            }
        }
    }

    /// <summary>
    /// 选取颜色
    /// </summary>
    /// <param name="x"></param>
    /// <param name="y"></param>
    private void PickColor(float x, float y)
    {
        selectedColor = colorPickTex.GetPixel((int)(x * colorPickTex.width), (int)(y * colorPickTex.height));//获取选择的颜色 
        selectedColorDisplay.color = selectedColor;//把选择的颜色展示出来
        c = Enumerable.Repeat<Color>(selectedColor, c.Length).ToArray<Color>();
    }

    /// <summary>
    /// 获取光标相对于指定rect的位置
    /// </summary>
    /// <param name="rect">指定的rect</param>
    /// <returns></returns>
    private Vector2 GetRelativePosition(Rect rect)
    {
        Vector2 cursorPos = computer.Canvas.transform.InverseTransformPoint(computer.cursorController.transform.position);
        Vector2 result = cursorPos - rect.position;
        result.x /= rect.width;
        result.y /= rect.height;
        return result;
    }

    /// <summary>
    /// 鼠标点击 
    /// </summary>
    private void OnMouseClick()
    {
        Vector2 relativeDrawPosition = GetRelativePosition(pictureRect);
        Vector2 relativePickPosition = GetRelativePosition(colorPickerRect);
        isInDrawArea = relativeDrawPosition.x >= 0f && relativeDrawPosition.x < 1f && relativeDrawPosition.y >= 0f && relativeDrawPosition.y < 1f;
        isInPickArea = relativePickPosition.x >= 0f && relativePickPosition.x < 1f && relativePickPosition.y >= 0f && relativePickPosition.y < 1f;
        if (isInPickArea || isInDrawArea)
        {
            canOperate = true;
        }
    }

    /// <summary>
    /// 鼠标点击后抬起
    /// </summary>
    private void OnMouseUnClick()
    {
        canOperate = false;
    }

    /// <summary>
    /// 是否打开画图程序 
    /// </summary>
    /// <param name="isActive"></param>
    public void Show(bool isActive)
    {
        gameObject.SetActive(isActive);
    }

    /// <summary>
    /// 画图
    /// </summary>
    /// <param name="relX"></param>
    /// <param name="relY"></param>
    private void SetPixel(float relX, float relY)
    {
        Color[] pixels = pictureTex.GetPixels();
        int num = Mathf.Clamp((int)(relX * (float)pictureWidth), 0, pictureWidth - 1);
        int num2 = Mathf.Clamp((int)(relY * (float)pictureHeight), 0, pictureHeight - 1);
        if (pixels[num2 * pictureWidth + num] != selectedColor)
        {
            pixels[num2 * pictureWidth + num] = selectedColor;
            pictureTex.SetPixels(pixels);
            pictureTex.Apply();
        }
      //pictureTex.SetPixels(num, num2, (int)lineSize.x, (int)lineSize.y, c);
    }

    /// <summary>
    /// 重置图片
    /// </summary>
    /// <param name="color"></param>
    private void ResetPixes(Color color)
    {
        Color[] array = new Color[pictureWidth * pictureHeight];
        for (int i = 0; i < array.Length; i++)
        {
            array[i] = color;
        }
        pictureTex.SetPixels(array);
        pictureTex.Apply();
    }
}

 最后为PaintProgramIcon的Button组件添加OnClick事件;

最后附上工程：链接：https://pan.baidu.com/s/1jJkaVEu 密码：mdwh