网页爬虫抓取URL简单实现

关键字：网页爬虫抓取URL简单实现 .

//开始......

package com.ogilvy.sayes.util;

import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Hashtable;

/*
Description: 爬网页用
Author : long.tang
*/

public class SearchCrawler {

public String myGetHttpFile2(String url) {

String urlSource = url;
StringBuffer htmlBuffer = new StringBuffer();
String returnStr = null;
try {
InputStream imageSource = new URL(urlSource).openStream();
int ch;
while ((ch = imageSource.read()) > -1) {
htmlBuffer.append((char) ch);
}
imageSource.close();
returnStr = new String(htmlBuffer);
returnStr = new String(returnStr.getBytes("ISO8859_1"), "GBK");
} catch (Exception e) {
System.out.println("error>>>>");
e.printStackTrace();
}

//System.out.println("@@@:" + returnStr);
if (returnStr != null) {
return returnStr;
} else {
return "nothing";
}

}

public void doit(String content, int depth) throws Exception {

depth--;
if (depth < 1) {
//System.out.println("break::::");
return;
}

SearchCrawler search = new SearchCrawler();
ArrayList list = new ArrayList();
int j = 0;
String start = "href=";
String end = "\"";
String url = "";
String type = "http";
String[] urls;
while (content.indexOf(start, j) > -1) {

url = content.substring(content.indexOf(start, j) + 6, content.indexOf(end, content.indexOf(start, j) + 6));//+6 href="
if (url.indexOf(type) > -1) {
if (url.indexOf(".css") == -1&&url.indexOf(".ico") == -1&&url.indexOf(".exe") == -1) {
System.out.println(url);

list.add(url);

if (list != null && list.size() > 0) {

for (int k = 0; k < list.size(); k++) {
doit(search.myGetHttpFile2(String.valueOf(list.get(k))), depth);

}

}
}

}


j = content.indexOf(start, j) + 1;

}

}

public static void main(String arg[]) {

SearchCrawler search = new SearchCrawler();
try {
search.doit(search.myGetHttpFile2("http://www.2345.com/"),3);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

}

}



//结束.....