HttpClient

HttpClient 是Apache Jakarta Common 下的子项目，可以用来提供高效的、最新的、功能丰富的支持 HTTP 协议的客户端编程工具包，并且它支持 HTTP 协议最新的版本和建议

pom.xml

        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.13</version>
        </dependency>

很多和发送请求相关的包会依赖此依赖，例如：阿里OSS包依赖了此依赖

发送请求步骤：

• 创建HttpClient对象
• 创建Http请求对象
• 调用HttpClient的execute方法发送请求，获取HttpResponse返回对象
• 获取HttpEntity返回内容
• 关闭资源

示例代码

HttpClientTest.java

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.json.JSONObject;

import java.io.IOException;
import java.io.UnsupportedEncodingException;

public class HttpClientTest {

    public static void main(String[] args) throws IOException {
        HttpClientTest httpClientTest = new HttpClientTest();

        // 发送get请求
        httpClientTest.testGet();

        // 发送post请求
        httpClientTest.testPost();
    }

    // 设置请求头
    public void setHeader(HttpRequestBase request) {
        request.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36");
    }

    // 设置请求体
    public StringEntity getEntity() throws UnsupportedEncodingException {
        JSONObject jsonObject = new JSONObject();
        jsonObject.put("pageNo", "1");
        jsonObject.put("pageSize", "2");

        StringEntity entity = new StringEntity(jsonObject.toString());
        // 指定请求的编码方式
        entity.setContentEncoding("utf-8");
        // 指定返回的数据格式
        entity.setContentType("application/json");

        return entity;
    }

    public void testGet() throws IOException {

        // 创建httpClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 创建请求对象(Get请求)
        HttpGet httpGet = new HttpGet("https://api.hunan.gov.cn/search/common/search/71043");
        setHeader(httpGet);

        // 发送请求，接受请求对象
        CloseableHttpResponse response = httpClient.execute(httpGet);

        // 获取服务端返回的状态码
        int statusCode = response.getStatusLine().getStatusCode();
        System.out.println("服务端返回的状态码为：" + statusCode);

        // 获取返回内容
        HttpEntity entity = response.getEntity();
        String body = EntityUtils.toString(entity);
        System.out.println("服务端返回的数据为：" + body);

        // 关闭资源
        response.close();
        httpClient.close();
    }

    public void testPost() throws IOException {

        // 创建httpClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 创建请求对象(Post请求)
        HttpPost httpPost = new HttpPost("https://www.jkqcyl.com/prod-api/notice/list");
        setHeader(httpPost);
        httpPost.setEntity(getEntity());

        // 发送请求，接受请求对象
        CloseableHttpResponse response = httpClient.execute(httpPost);

        // 获取服务端返回的状态码
        int statusCode = response.getStatusLine().getStatusCode();
        System.out.println("服务端返回的状态码为：" + statusCode);

        // 获取返回内容
        HttpEntity entity = response.getEntity();
        String body = EntityUtils.toString(entity);
        System.out.println("服务端返回的数据为：" + body);

        // 关闭资源
        response.close();
        httpClient.close();
    }
}

Java爬虫

HoldcgController.java

import com.alibaba.fastjson2.JSONObject;
import com.ruoyi.jkqcyl.constants.HoldcgConstant;
import com.ruoyi.web.controller.common.R;
import org.apache.http.Consts;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.springframework.web.bind.annotation.*;
import java.io.IOException;
import java.util.HashMap;

import java.util.Map;

/**
 * 对接荷云彩平台
 * www.holdcg.com
 */
@RestController
@RequestMapping("/holdcg")
public class HoldcgController {

    private static final CloseableHttpClient httpclient = HttpClients.createDefault();

    /**
     * 获取招采信息
     * https://www.holdcg.com/cms/bulletin/getAll.do
     * POST
     * @param data:
     * {
     *     "pageNo": "1",
     *     "pageSize": "16",
     *     "purchaseMode": "10"
     * }
     *
     * 发送给荷云彩的请求体
     * 'page': {
     *      'current': 0, 当前页
     *      'size': 16, 每页条数
     * },
     * 'cmsBulletinDTO': {
     *     'type': 'all',
     *     'purchaseMode': '', 空代表全部,10代表招标信息,-10代表非招标信息
     *     'loginCompanyId': -1,
     * }
     * @return
     */
    @PostMapping("/bulletin")
    public R getAll(@RequestBody Map<String, String> data) {
        HttpPost httpPost = new HttpPost(HoldcgConstant.getBulletinUrl());
        unityHttpHeader(httpPost);

        Map<String, Object> requestBody = new HashMap<>();
        generatePageInfo(requestBody, Integer.parseInt(data.get("pageNo")) - 1, Integer.parseInt(data.get("pageSize")));
        generateCmsBulletinDTOInfo(requestBody, "10".equals(data.get("purchaseMode")));
        httpPost.setEntity(httpJsonEntity(requestBody));

        String result = null;
        // Java中的try-with-resources语句，里面可以写多个需要关闭的资源，免去写手动调用response.close()方法来关闭HTTP响应
        try (CloseableHttpResponse response = httpclient.execute(httpPost)) {
            result = EntityUtils.toString(response.getEntity());
        } catch (ClientProtocolException e) {
            throw new RuntimeException("网络卡顿，请稍后再试");
        } catch (IOException e) {
            throw new RuntimeException("参数错误");
        }

        return R.success(JSONObject.parseObject(result, Map.class).get("result"));
    }

    public Object innerGetAll(Map<String, String> data) {
        if (!data.containsKey("pageNo")) {
            data.put("pageNo", "1");
        }
        if (!data.containsKey("pageSize")) {
            data.put("pageSize", "16");
        }
        return getAll(data).getData();
    }

    /**
     * 请求体添加分页信息(page)
     * @param requestEntity
     * @param current
     * @param size
     */
    private void generatePageInfo(Map<String, Object> requestEntity, int current, int size) {
        Map<String, Object> pageMap = new HashMap<>();
        pageMap.put("current", current);
        pageMap.put("size", size);
        requestEntity.put("page", pageMap);
    }

    /**
     * 添加具体查询条件信息(cmsBulletinDTO)
     * @param requestEntity
     * @param isTender
     */
    private void generateCmsBulletinDTOInfo(Map<String, Object> requestEntity, boolean isTender) {
        Map<String, Object> cmsBulletinDTOMap = new HashMap<>();
        cmsBulletinDTOMap.put("type", "all");
        cmsBulletinDTOMap.put("loginCompanyId", -1);
        cmsBulletinDTOMap.put("purchaseMode", isTender ? "10" : "-10");
        requestEntity.put("cmsBulletinDTO", cmsBulletinDTOMap);
    }

    /**
     * 设置统一请求头
     * @param request
     */
    private void unityHttpHeader(HttpRequestBase request) {
        request.setHeader("Accept", "application/json, text/plain, */*");
        request.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
        request.setHeader("Connection", "keep-alive");
        request.setHeader("Content-Type", "application/json");
        request.setHeader("Cookie", "Hm_lvt_8f5cb8c05625f4f4172a771ed25ce038=1690960776; SESSION=3092a88d-e227-4192-b51a-d83f76d2e59d");
        request.setHeader("Origin", "https://www.holdcg.com");
        request.setHeader("Referer", "https://www.holdcg.com/");
        request.setHeader("Sec-Fetch-Dest", "empty");
        request.setHeader("Sec-Fetch-Mode", "cors");
        request.setHeader("Sec-Fetch-Site", "same-origin");
        request.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36");
        request.setHeader("sec-ch-ua", "\"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"");
        request.setHeader("sec-ch-ua-mobile", "?0");
        request.setHeader("sec-ch-ua-platform", "\"Windows\"");
    }

    /**
     * 将一个map转成http可以识别的实体
     * @param requestEntity
     * @return
     */
    private StringEntity httpJsonEntity(Map<String, Object> requestEntity) {
        JSONObject json = new JSONObject(requestEntity);
        StringEntity entity = new StringEntity(json.toString(), Consts.UTF_8);
        return entity;
    }
}