Leo's Blog

Welcome to Leo's World

0%

原创： httpclient 4.0 使用 - 访问开心网的各种组件 例子

发表于 更新于 分类于 Valine：
本文字数： 12k 阅读时长 ≈ 11 分钟

在网上查了好长时间，发现httpclient的例子都是讲httpclient 3.X的。httpclient 4.0的例子简直是凤毛麟角。只好自己上官网仔细了查了下文档和官网示例，受益匪浅啊！
由于前段时间对“开心网”的“开心大亨”很感兴趣，但是每隔一段时间就需要看一下最新的价格变化，很是麻烦，由于我就想起用httpclient写个程序，自动查看最新的价格，并和前次的价格做比较，好让我可以选择利润最大的进行投资。

程序用到的包如下，为了附件的瘦身，请大家自己下载啊，附件中是没有这些lib的啊。
commons-collections-3.2.1.jar
commons-configuration-1.6.jar
commons-io-1.4.jar
commons-lang-2.4.jar
commons-logging-1.1.1.jar
httpclient-4.0.jar
httpcore-4.0.1.jar

由于只是例子，而且httpclient4.0的结构变化很大，自己对其理解也不是很深，此程序只是个入门程序，如果有高手在的话，希望大家可以提出宝贵的意见。
程序写的不是很美观，但意在让大家明白如何简单的使用httpclient4.0，所以没有对程序做任何的重构的整理，望大家多多谅解啊。

程序代码如下：

主程序： KaiXin001.java

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
/*
* 2009/12/21
* Author:  Yuan Hongzhi 
*/
import java.io.File;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.cookie.Cookie;
import org.apache.http.entity.BufferedHttpEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HTTP;
import org.apache.http.util.EntityUtils;
public class KaiXin001 {
  public InputStream getResourceAsStream(String filename) throws Exception {
    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
    InputStream in = null;
    if (classLoader != null) {
      in = classLoader.getResourceAsStream(filename);
    }
    if (in == null) {
      in = ClassLoader.getSystemResourceAsStream(filename);
    }
    if (in == null) {
      throw new Exception("Can't find resource file: " + filename);
    } else {
      return in;
    }
  }
  public void writeToFile(String file, HttpEntity entity) throws Exception {
    writeToFile(file, EntityUtils.toString(entity));
  }
  public void writeToFile(String file, String data) throws Exception {
    FileUtils.writeStringToFile(new File(file), data, "UTF-8");
  }
  public String getContent(HttpEntity entity) throws Exception {
    if (entity != null) {
      entity = new BufferedHttpEntity(entity);
      long len = entity.getContentLength();
      System.out.println("Length: " + len);
      System.out.println("====================\r\n");
      return EntityUtils.toString(entity, "UTF-8");
    } else {
      System.out.println("entity is null.");
      return null;
    }
  }
  public void setCookie(DefaultHttpClient httpclient, List<Cookie> cookies) {
    if (cookies.isEmpty()) {
      System.out.println("Cookie is empty.");
      return;
    } else {
      for (int i = 0; i < cookies.size(); i++) {
        System.out.println((i + 1) + " - " + cookies.get(i).toString());
        httpclient.getCookieStore().addCookie(cookies.get(i));
      }
      System.out.println();
    }
  }
// "开心网其它组件URL如下，大家可以添加上自己喜欢的组件URL。"
// "http://www.kaixin001.com/!slave/index.php", "朋友买卖"
// "http://www.kaixin001.com/!parking/index.php", "争车位"
// "http://www.kaixin001.com/!house/index.php?_lgmode=pri", "买房子"
// "http://www.kaixin001.com/!house/index.php?_lgmode=pri&t=49"
// "http://www.kaixin001.com/!house/garden/index.php","花园"
// "http://www.kaixin001.com/!rich/market.php", "超级大亨"
  public String enterComponentContent(String url, String componentName,
      DefaultHttpClient httpclient, List<Cookie> cookies,
      HttpResponse response, HttpEntity entity) throws Exception {
    System.out.println("--- Enter: " + componentName + " ---");
    System.out.println("--- Url:   " + url + " ---");
    setCookie(httpclient, cookies);
    HttpGet httpget = new HttpGet(url);
    response = httpclient.execute(httpget);
    entity = response.getEntity();
    return getContent(entity);
  }
  public void showResponseStatus(HttpResponse response) {
// System.out.println(response.getProtocolVersion());
// System.out.println(response.getStatusLine().getStatusCode());
// System.out.println(response.getStatusLine().getReasonPhrase());
    System.out.println(response.getStatusLine().toString());
    System.out.println("
---\r\n");
  }
  public static void main(String[] args) throws Exception {
    KaiXin001 kx = new KaiXin001();
    DefaultHttpClient httpclient = new DefaultHttpClient();
    HttpPost httpost = new HttpPost("http://www.kaixin001.com/login/login.php");
    List<NameValuePair> qparams = new ArrayList<NameValuePair>();
    qparams.add(new BasicNameValuePair("email", "email"));
    qparams.add(new BasicNameValuePair("password", "password"));
    httpost.setEntity(new UrlEncodedFormEntity(qparams, HTTP.UTF_8));
    HttpResponse response = httpclient.execute(httpost);
    kx.showResponseStatus(response);
// HttpResponse response = httpclient.execute(httpget);
    HttpEntity entity = response.getEntity();
    kx.getContent(entity);
// Login
    List<Cookie> cookies = httpclient.getCookieStore().getCookies();
    System.out.println("Post logon cookies:");
    kx.setCookie(httpclient, cookies);
// Redirect to home page
    String homepage = "http://www.kaixin001.com/home/";
    String content = null;
    content = kx.enterComponentContent(homepage, "Home page", httpclient,
        cookies, response, entity);
// Component
    String componet = "http://www.kaixin001.com/!rich/market.php";
    content = kx.enterComponentContent(componet, "Component", httpclient,
        cookies, response, entity);
// 
---
    kx.writeToFile("c:/kaixin.html", content);
// When HttpClient instance is no longer needed,
// shut down the connection manager to ensure
// immediate deallocation of all system resources
    httpclient.getConnectionManager().shutdown();
  }
}

访问开心大亨组件的程序：KaiXin_Rich.java

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
/*
* 2009/12/21
* Author:  Yuan Hongzhi 
*/
import java.awt.Toolkit;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.collections.keyvalue.DefaultKeyValue;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HTTP;
public class KaiXin_Rich extends KaiXin001 {
  public static List<DefaultKeyValue> curList = new ArrayList<DefaultKeyValue>();
  public static List<DefaultKeyValue> preList = new ArrayList<DefaultKeyValue>();
  public void copyList(List<DefaultKeyValue> src, List<DefaultKeyValue> des) {
    for (DefaultKeyValue dkv : src) {
      DefaultKeyValue kv = new DefaultKeyValue(dkv.getKey(), dkv.getValue());
      des.add(kv);
    }
  }
  public Double subNumber(Object string) {
    String rex = "([\\d\\.]*)";
    Pattern pattern = Pattern.compile(rex);
    Matcher match = pattern.matcher(string.toString());
    if (match.find()) {
      return Double.valueOf(match.group());
    } else
      return -1.0;
  }
  public String extractNoneAscii(Object string) {
    String rex = "([^\\x00-\\xff]+)";
    Pattern pattern = Pattern.compile(rex);
    Matcher match = pattern.matcher(string.toString());
    if (match.find()) {
      return match.group();
    } else
      return "";
  }
  public static void main(String[] args) throws Exception {
    KaiXin_Rich kx = new KaiXin_Rich();
    PropertiesConfiguration config = new PropertiesConfiguration();
    config.load(kx.getResourceAsStream("properties/kaixin001.properties"),
"UTF-8");
    DefaultHttpClient httpclient = new DefaultHttpClient();
    HttpPost httpost = new HttpPost("http://www.kaixin001.com/login/login.php");
    List<NameValuePair> qparams = new ArrayList<NameValuePair>();
    qparams.add(new BasicNameValuePair("email", config.getString("email")));
    qparams
        .add(new BasicNameValuePair("password", config.getString("password")));
    httpost.setEntity(new UrlEncodedFormEntity(qparams, HTTP.UTF_8));
    HttpResponse response = httpclient.execute(httpost);
    kx.showResponseStatus(response);
// HttpResponse response = httpclient.execute(httpget);
    HttpEntity entity = response.getEntity();
    kx.getContent(entity);
// Login
    List<Cookie> cookies = httpclient.getCookieStore().getCookies();
    System.out.println("Post logon cookies:");
    kx.setCookie(httpclient, cookies);
// Redirect to home page
    String homepage = "http://www.kaixin001.com/home/";
    String content = null;
    content = kx.enterComponentContent(homepage, "Home page", httpclient,
        cookies, response, entity);
    int loopTime = config.getInt("times");
    long sleepTime = config.getLong("period") * 60;
    for (int j = 0; j < loopTime; j++) {
// Component
      String componet = config.getString("url");
      content = kx.enterComponentContent(componet, "Component", httpclient,
          cookies, response, entity);
      int beginIndex = content.indexOf(config.getString("startRex"));
      int endIndex = content.lastIndexOf(config.getString("endRex"));
      content = content.substring(beginIndex, endIndex);
// 
---
      String rex = config.getString("displayKeyRex");
      Pattern pattern = Pattern.compile(rex, Pattern.MULTILINE);
      Matcher match = pattern.matcher(content);
      List<String> keys = new ArrayList<String>();
      String matched = null;
      while (match.find()) {
        matched = match.group();
        keys.add(matched);
      }
      rex = config.getString("displayValueRex");
      pattern = Pattern.compile(rex, Pattern.MULTILINE);
      match = pattern.matcher(content);
      List<String> values = new ArrayList<String>();
      while (match.find()) {
        matched = match.group();
        values.add(matched);
      }
      DefaultKeyValue kv = null;
      for (int i = 0; i < keys.size(); i++) {
        kv = new DefaultKeyValue(keys.get(i), values.get(i));
        curList.add(kv);
      }
      String rtnString = "";
      String eachLine = null;
      for (DefaultKeyValue dkv : curList) {
        eachLine = dkv.getKey() + "\t:" + dkv.getValue();
        rtnString += eachLine;
        rtnString += "\r\n";
        System.out.println(eachLine);
      }
      System.out.println("\r\n===== Changed =====\r\n");
      String allPriceString = "===== Changed =====\r\n\r\n";
      StringBuffer price = null;
      double changed = 0;
      boolean beepFlag = false;
      for (int i = 0; i < curList.size() && curList.size() == preList.size(); i++) {
        double curMoney = kx.subNumber(curList.get(i).getValue());
        double preMoney = kx.subNumber(preList.get(i).getValue());
        if ((curMoney - preMoney) > 1E-5) {
          changed = curMoney - preMoney;
          DecimalFormat format = new DecimalFormat("###.##");
          String number = format.format(changed);
          price = new StringBuffer();
          price.append(curList.get(i).getKey());
          price.append("\t: ");
          price.append(preList.get(i).getValue());
          price.append(" -> ");
          price.append(curList.get(i).getValue());
          price.append("\t");
          price.append(" Changed: ");
          price.append(number);
          price.append(kx.extractNoneAscii(curList.get(i).getValue()));
          System.out.println(price);
          allPriceString = allPriceString + price.toString() + "\r\n";
          beepFlag = true;
        }
      }
      if (beepFlag) {
        Toolkit.getDefaultToolkit().beep();
      }
      preList.clear();
// System.out.println(content);
      String dateTime = DateUtils
          .getSystemDateTime(DateTimePattern.DATE_TIME_LONG_UNFORMAT);
      kx.writeToFile("c:/kaixin001/price_" + dateTime + ".txt", rtnString
+ "\r\n" + allPriceString);
// kx.writeToFile("c:/kaixin.html", content);
      kx.copyList(curList, preList);
      curList.clear();
      Thread.sleep(sleepTime * 1000);
    }
// When HttpClient instance is no longer needed,
// shut down the connection manager to ensure
// immediate deallocation of all system resources
    httpclient.getConnectionManager().shutdown();
  }
}
坚持原创及高品质技术分享，您的支持将鼓励我继续创作！