爬虫获取公司通讯录

//修改UserAgent 模拟Chrome浏览器
private static Site site = Site.me()
        .setRetryTimes(2)
        .setSleepTime(100)
        .setCharset("UTF-8")
        .setUserAgent("Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36");
public static SimpleHttpClient simpleHttpClient = new SimpleHttpClient(site);
private static Request request = new Request();

/**
 * @return 识别成功的验证码
 * @throws Exception
 */
public static void login() throws Exception {
    request.setCharset("gbk");
    request.setUrl(SPIDER_CAPTCHA_URL);
    //获取验证码图片
    Page captchaPage = simpleHttpClient.get(request);
    //图片转文本
    String captchaString = getCaptcha(captchaPage.getBytes());

    logger.debug("本次识别的验证码: " + captchaString);
    request.setUrl(SPIDER_LOGIN_URL);
    HttpRequestBody httpRequestBody = new HttpRequestBody();
    httpRequestBody.setBody(("clientid=0001&userid=" + SPIDER_USERNAME + "&password=" + SPIDER_PASSWORD + "&yzm=" + captchaString + "&Submit=%CC%E1%BD%BB").getBytes("utf-8"));
    request.setRequestBody(httpRequestBody);
    request.setMethod("post");

    Page loginResult = simpleHttpClient.get(request);

    System.out.println(loginResult.getRawText());
}

private static CloseableHttpClient httpClient = HttpClients.createDefault();
public static void login() throws Exception {

    HttpGet httpGet = new HttpGet(SPIDER_CAPTCHA_URL);
    CloseableHttpResponse response1 = null;

    try {
        response1 = httpClient.execute(httpGet);
        HttpEntity entity1 = response1.getEntity();
        byte[] bytes = EntityUtils.toByteArray(entity1);
        //图片转文本
        String captchaString = getCaptcha(bytes);

        logger.debug("本次识别的验证码: " + captchaString);

        EntityUtils.consume(entity1);
        
    } finally {
        try {
            if (response1 != null) {
                response1.close();
            }
            
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

HttpPost httpPost = new HttpPost(SPIDER_LOGIN_URL);
CloseableHttpResponse response2 = null;

try {
    
    List<NameValuePair> parameters = new ArrayList<>();
    parameters.add(new BasicNameValuePair("clientid","0001"));
    parameters.add(new BasicNameValuePair("userid",SPIDER_USERNAME));
    parameters.add(new BasicNameValuePair("password",SPIDER_PASSWORD));
    parameters.add(new BasicNameValuePair("yzm",captchaString));
    parameters.add(new BasicNameValuePair("Submit", new String("提交".getBytes("utf-8"),"gbk")));
    
    UrlEncodedFormEntity urlEncodedFormEntity = new UrlEncodedFormEntity(parameters);
    httpPost.setEntity(urlEncodedFormEntity);
    response2 = httpClient.execute(httpPost);
    HttpEntity entity2 = response2.getEntity();
    Header[] allHeaders = response2.getAllHeaders();
    for (Header allHeader : allHeaders) {
        System.out.println(allHeader);
    }
    EntityUtils.consume(entity2);
    if(response2.getStatusLine().getStatusCode()!=302){
        throw new Exception("模拟登录失败");
    }
    logger.info("模拟登录成功!!");

} finally {
    try {
        if (response2 != null) {
            response2.close();
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
}

private static CloseableHttpClient httpClient = HttpClients.createDefault();
 ↓
private static CloseableHttpClient httpClient = HttpClients.custom()
        .disableAutomaticRetries() //关闭自动处理重定向
        .setRedirectStrategy(new LaxRedirectStrategy()).build();//利用LaxRedirectStrategy处理POST重定向问题

//get方法 封装了每次发请求的细节 返回报文正文 
String first = get(SPIDER_TEL_URL);
//使用jsoup解析 html
Document document = Jsoup.parse(first);
//使用xpath 定位下拉框 获取 option元素集合
Elements select = document.select("select[name='deptid'] > option");
List<Map> list = new ArrayList<>();
for (Element element : select) {
    //根据每一个option的value 发请求 获取每个部门的员工信息 保存到list中
    searchDept(element.val(),list);
}
list.forEach(System.out::println);
//使用gson 序列化list
String json = new Gson().toJson(list);

System.out.println(json);

private static void searchDept(String deptId,List list) throws Exception{
    String html = get(SPIDER_TEL2_URL + deptId);
    Document document = Jsoup.parse(html);
    Elements select1 = document.select("table:has(table) table tbody tr");
    for (Element element : select1) {
        Map map = new LinkedHashMap();

        Elements children = element.children();
        for (int i = 1; i <= children.size(); i++) {
            String s = mapTitle.get(i);
            if (s != null) {
                map.put(s, element.child(i - 1).text());
            }
        }
        list.add(map);
    }
}

private static RequestConfig requestConfig = RequestConfig.custom()
        .setConnectionRequestTimeout(1000)//从连接池获取连接的超时时间
        .setSocketTimeout(5000)//读超时时间
//            .setProxy(new HttpHost("127.0.0.1",8888))//设置代理 fiddler 抓包调试
        .setConnectTimeout(5000).build();//连接超时时间
        
httpGet.setConfig(requestConfig);

int i = 1;
while (i <= 5) {
    try {
        logger.info(String.format("开始第%d次 模拟登录", i));
        login1();
        break;
    } catch (Exception e) {
        e.printStackTrace();
        i++;
    }
}