c# 爬虫 -ChromeDriver+HtmlAgilityPack爬取比赛实时比分

Posted 2022-02-10 dotNET跨平台

背景

最近NBA总决赛吸引了不少球迷，但是因为时差的关系，人家在比赛，我们在上班，有时候上班又不好意思光明正大的看比赛，那有什么办法 可以光明正大的看又不被发现呢。

有，自己动手丰衣足食，ChromeDriver+htmlAgilityPack爬取比赛实时比分。

概述

WebDriver 就是对浏览器提供的原生API进行封装，使其成为一套更加面向对象的Selenium WebDriver API。使用这套API可以操控浏览器的开启、关闭，打开网页，操作界面元素，控制Cookie，还可以操作浏览器截屏、安装插件、设置代理、配置证书等。

HtmlAgilityPack是.net下的一个HTML解析类库。支持用XPath来解析HTML。这个意义不小，为什么呢？因为对于页面上的元素的xpath某些强大的浏览器能够直接获取得到，并不需要手动写。节约了大半写正则表达式的时间，当然正则表达式有时候在进一步获取的时候还需要写，但是通过xpath解析之后，正则表达式已经要匹配的范围已经非常小了。而且，不用正则表达式在整个页面源代码上匹配，速度也会有提升。总而言之，通过该类库，先通过浏览器获取到xpath获取到节点内容然后再通过正则表达式匹配到所需要的内容，无论是开发速度，还是运行效率都有提升。

代码实现

下面我们来看下如何使用ChromeDriver+HtmlAgilityPack爬取比赛实时比分。

抓取网页

//api
                        string url = "https://sports.qq.com/kbsweb/game.htm?mid=" + model.TxMatchId;

                        //   string url = "https://kbs.sports.qq.com/kbsweb/game.htm?mid=100000:55370464";
                        //  MatchUpdate model = new MatchUpdate();

                        var cds = ChromeDriverService.CreateDefaultService();
                        //是否应隐藏服务的命令提示符窗口
                        cds.HideCommandPromptWindow = true;

                        ChromeOptions options = new ChromeOptions();
                        options.AddArguments("--test-type", "--ignore-certificate-errors");
                        //   options.AddArguments("user-agent=mozilla/5.0 (linux; u; android 2.3.3; en-us; sdk build/ gri34) applewebkit/533.1 (khtml, like gecko) version/4.0 mobile safari/533.1");
                        options.AddArgument("enable-automation");

                        //  options.setBinary("C:/Program Files (x86)/Google/Chrome/chrome.exe");

                        var r = ZhimaHttpProxy.GetProxy(false);
                        if (r != null)
                        
                            Console.WriteLine(r.Address.Host);
                            Console.WriteLine(r.Address.Port);

                            string proxy_Host = r.Address.Host;
                            int proxy_Post = r.Address.Port;
                            string Ex_Proxy_Name = "proxy.zip";

                            options.Proxy = null;
                            options.AddArguments("--proxy-server=" + proxy_Host + ":" + proxy_Post.ToString());
                            options.AddExtension(Ex_Proxy_Name);
                        

                        if (IsHideMode)
                            options.AddArgument("headless");

                        string dic = System.Environment.CurrentDirectory + "\\\\cos";

                        if (IsHideMode)
                        
                            using (IWebDriver driver = new OpenQA.Selenium.Chrome.ChromeDriver(cds, options, TimeSpan.FromSeconds(120)))

                            
                                Excule(driver, url, model);
                            
                        
                        else
                        
                            using (IWebDriver driver = new OpenQA.Selenium.Chrome.ChromeDriver(dic, options, TimeSpan.FromSeconds(120)))

                            
                                Excule(driver, url, model);
                            
                        

解析网页

HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
            doc.LoadHtml(Helper.ReadTxt(System.Environment.CurrentDirectory + "\\\\PageSource\\\\" + info.TxMatchId.Replace(":", "_") + ".txt"));
            int? HomeTeamScore = null;
            int? GuestTeamScore = null;
            List<MatchScore> MatchScoreList = new List<MatchScore>();
            HtmlNode titleNodes = doc.DocumentNode.SelectSingleNode("//div[@class='inner']");
            var titleNodes2 = doc.DocumentNode.SelectSingleNode("//div[@class='content-wrapper']");
            if (titleNodes2 != null)
            //  if (false)
            
                Console.WriteLine(titleNodes2.InnerText);
                var host = titleNodes2.SelectSingleNode("//div[@class='team-goal host']");
                var arr = host.InnerText.Replace("\\r\\n", "|");
                var arrs = arr.Split('|');
                arrs = arrs.Where(o => !string.IsNullOrWhiteSpace(o)).Select(o => o.Split('(')[0].Trim()).ToArray();
                HomeTeamScore = int.Parse(arrs[1]);
                MatchScoreList.Add(new MatchScore()  TeamName = arrs[0], TeamScore = HomeTeamScore );

                var guest = titleNodes2.SelectSingleNode("//div[@class='team-goal guest']");

                var arr2 = guest.InnerText.Replace("\\r\\n", "|");
                var arrs2 = arr2.Split('|');
                arrs2 = arrs2.Where(o => !string.IsNullOrWhiteSpace(o)).Select(o => o.Split('(')[0].Trim()).ToArray();

                GuestTeamScore = int.Parse(arrs2[1]);

                MatchScoreList.Add(new MatchScore()  TeamName = arrs2[0], TeamScore = GuestTeamScore );
            
            else
            
                var a = titleNodes.SelectNodes("//a[@data-target='teamName']");
                var score = titleNodes.SelectNodes("//span[@class='score']");
                if (score != null)
                
                    int i = 0;
                    foreach (var item in score)
                    
                        SetText("\\r\\n" + item.InnerText?.Trim());
                        if (i == 0)
                        
                            HomeTeamScore = Convert.ToInt32(item.InnerText?.Trim());
                        
                        if (i == 1)
                        
                            GuestTeamScore = Convert.ToInt32(item.InnerText?.Trim());
                        

                        i++;
                    
                

                if (a != null)
                
                    int i = 0;
                    foreach (var item in a)
                    
                        SetText("\\r\\n" + item.InnerText?.Trim());

                        if (i == 0)
                        
                            MatchScoreList.Add(new MatchScore()  TeamName = item.InnerText?.Split('(')[0].Trim(), TeamScore = HomeTeamScore );
                        
                        if (i == 1)
                        
                            MatchScoreList.Add(new MatchScore()  TeamName = item.InnerText?.Split('(')[0].Trim(), TeamScore = GuestTeamScore );
                        

                        i++;
                    
                
            

            string statusstr = "";

            var t = titleNodes.SelectSingleNode("//div[@class='datetime-live-desc']");
            SetText("\\r\\n" + t.InnerText?.Trim());
            statusstr = t.InnerText?.Trim();

            var Status = info.Status;
            if (!string.IsNullOrWhiteSpace(statusstr) && statusstr.Contains("已结束"))
            
                Status = MatchStatus.End;
            

最后再搞个小弹框，把数据输出即可。