C#和HttpClient结合示例：微博热点数据分析

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace WeiboCrawler

{

class Program

{

// 创建一个HTTP客户端对象，用于发送请求和接收响应

static HttpClient httpClient = new HttpClient();
    // 创建一个数据结构，用于保存热点话题的信息
    static List<Topic> topics = new List<Topic>();

    // 定义一个锁对象，用于同步多线程操作
    static object locker = new object();

    static void Main(string[] args)
    {
        // 亿牛云爬虫标准版，使用代理服务设置代理域名、端口、用户名和密码
        var proxy = new WebProxy("http://wwww.16yun.cn:8080");
        proxy.Credentials = new NetworkCredential("16YUNXXX", "16IPXXX");
        httpClient.DefaultRequestHeaders.Add("Proxy-Authorization", "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes("username:password")));
        httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36");
        httpClient.Timeout = TimeSpan.FromSeconds(10);

        // 定义一个热点话题的URL列表，每个URL对应一个热点话题的页面
        var urls = new List<string>
        {
            "https://s.weibo.com/top/summary?cate=realtimehot",
            "https://s.weibo.com/top/summary?cate=socialevent",
            "https://s.weibo.com/top/summary?cate=entertainment",
            "https://s.weibo.com/top/summary?cate=sports",
            "https://s.weibo.com/top/summary?cate=tech",
            "https://s.weibo.com/top/summary?cate=finance"
        };

        // 使用多线程技术，创建多个线程，每个线程负责爬取一个热点话题的数据
        var tasks = new List<Task>();
        foreach (var url in urls)
        {
            tasks.Add(Task.Run(() => Crawl(url)));
        }

        // 等待所有线程完成任务
        Task.WaitAll(tasks.ToArray());

        // 使用LINQ或者其他方法，简单的对数据进行排序、分组操作，得到一些有意义的统计结果
        Console.WriteLine("爬取完成，共获取了{0}个热点话题的信息。", topics.Count);
        Console.WriteLine("按阅读量降序排列的前10个热点话题如下：");
        foreach (var topic in topics.OrderByDescending(t => t.ReadCount).Take(10))
        {
            Console.WriteLine("{0} {1} {2} {3}", topic.Title, topic.Link, topic.ReadCount, topic.DiscussCount);
        }
        Console.WriteLine("按讨论量降序排列的前10个热点话题如下：");
        foreach (var topic in topics.OrderByDescending(t => t.DiscussCount).Take(10))
        {
            Console.WriteLine("{0} {1} {2} {3}", topic.Title, topic.Link, topic.ReadCount, topic.DiscussCount);
        }
        Console.WriteLine("按类别分组的热点话题数量如下：");
        foreach (var group in topics.GroupBy(t => t.Category))
        {
            Console.WriteLine("{0} {1}", group.Key, group.Count());
        }
    }

    // 定义一个方法，用于爬取一个热点话题的数据
    static void Crawl(string url)
    {
        try
        {
            // 发送GET请求，获取响应内容
            var response = httpClient.GetAsync(url).Result;
            var content = response.Content.ReadAsStringAsync().Result;

            // 使用正则表达式或者HTML解析器，从响应内容中提取热点话题的标题、链接、阅读量、讨论量等信息，并保存到一个数据结构中
            var regex = new Regex(@"<td class=""td-02""><a href=""(?<link>.+?)"" target=""_blank"" title=""(?<title>.+?)"">.+?</a><span>(?<readcount>\d+)</span></td>\s*<td class=""td-03""><i class=""icon-txt"">.+?</i><span>(?<discusscount>\d+)</span></td>");
            var matches = regex.Matches(content);
            foreach (Match match in matches)
            {
                var topic = new Topic
                {
                    Title = match.Groups["title"].Value,
                    Link = "https://s.weibo.com" + match.Groups["link"].Value,
                    ReadCount = int.Parse(match.Groups["readcount"].Value),
                    DiscussCount = int.Parse(match.Groups["discusscount"].Value),
                    Category = url.Split('=')[1]
                };
                // 使用锁对象，避免多线程操作数据结构时发生冲突
                lock (locker)
                {
                    topics.Add(topic);
                }
            }
        }
        catch (Exception ex)
        {
            // 处理异常情况，如网络超时、响应格式错误等
            Console.WriteLine("爬取{0}时发生错误：{1}", url, ex.Message);
        }
    }
}

// 定义一个类，用于表示一个热点话题的信息
class Topic
{
    public string Title { get; set; } // 标题
    public string Link { get; set; } // 链接
    public int ReadCount { get; set; } // 阅读量
    public int DiscussCount { get; set; } // 讨论量
    public string Category { get; set; } // 类别
}

}