HttpUrlConnection底层实现和关于java host绑定ip即时生效的设置及分析
? ? public static String getResponseText(String queryUrl,String host,String ip) { //queryUrl，完整的url，host和ip需要绑定的host和ip
?????? InputStream is = null;
?????? BufferedReader br = null;
?????? StringBuffer res = new StringBuffer();
?????? try {
?????????? HttpURLConnection httpUrlConn = null;
?????????? URL url = new URL(queryUrl);
?????????? if(ip!=null){
?????????? ??? String str[] = ip.split("\\.");
?????????? ??? byte[] b =new byte[str.length];
?????????? ??? for(int i=0,len=str.length;i<len;i++){
?????????? ??????? b[i] = (byte)(Integer.parseInt(str[i],10));
?????????? ??? }
??? ??????????? Proxy proxy = new Proxy(Proxy.Type.HTTP,
??? ??????????? new InetSocketAddress(InetAddress.getByAddress(b), 80));? //b是绑定的ip，生成proxy代理对象，因为http底层是socket实现，
??? ??????????? httpUrlConn = (HttpURLConnection) url
??????????????? .openConnection(proxy);
?????????? }else{
??? ??????????? httpUrlConn = (HttpURLConnection) url
??? ??????????????????? .openConnection();?
?????????? }
?????????? httpUrlConn.setRequestMethod("GET");
?????????? httpUrlConn.setDoOutput(true);
?????????? httpUrlConn.setConnectTimeout(2000);
?????????? httpUrlConn.setReadTimeout(2000);
?????????? httpUrlConn.setDefaultUseCaches(false);
?????????? httpUrlConn.setUseCaches(false);
?
httpUrlConn = (HttpURLConnection) url.openConnection(proxy)
?
java.net.URL类里面的openConnection方法：
public URLConnection openConnection(Proxy proxy){
?? …
?? return handler.openConnection(this, proxy); Handler是sun.net.www.protocol.http.Handler.java类，继承java.net. URLStreamHandler.java类，用来处理http连接请求响应的。
}
?
Handler的方法：
protected java.net.URLConnection openConnection(URL u, Proxy p)
??????? throws IOException {
??????? return new HttpURLConnection(u, p, this);
??? }
?
只是简单的生成sun.net.www.protocl.http.HttpURLConnection对象，并进行初始化
protected HttpURLConnection(URL u, Proxy p, Handler handler) {
??????? super(u);
??????? requests = new MessageHeader();? 请求头信息生成类
??????? responses = new MessageHeader(); 响应头信息解析类
??????? this.handler = handler;?
????????instProxy = p;? 代理服务器对象
??????? cookieHandler = (CookieHandler)java.security.AccessController.doPrivileged(
??????????? new java.security.PrivilegedAction() {
??????????? public Object run() {
??????????????? return CookieHandler.getDefault();
??????????? }
??????? });
??????? cacheHandler = (ResponseCache)java.security.AccessController.doPrivileged(
??????????? new java.security.PrivilegedAction() {
??????????? public Object run() {
??????????????? return ResponseCache.getDefault();
??????????? }
??????? });
??? }
??
sun.net.www.protocl.http.HttpURLConnection.java的getInputStream方法：
?
public synchronized InputStream getInputStream() throws IOException {
???
?????...socket连接
???? connect();
???? ...
???? ps = (PrintStream)http.getOutputStream(); 获得输出流，打开连接之后已经生成。
?
?????? if (!streaming()) {
???????????? writeRequests(); ?输出http请求头信息
?????? }
???? ...
???? http.parseHTTP(responses, pi, this);? 解析响应信息
??????????????? if(logger.isLoggable(Level.FINEST)) {
??????????????????? logger.fine(responses.toString());
??????????????? }
??????????????? inputStream = http.getInputStream();? 获得输入流
其中connect()调用方法链：
plainConnect(){
...
??????????????? Proxy p = null;
??????????????? if (sel != null) {
??????????????????? URI uri = sun.net.www.ParseUtil.toURI(url);
??????????????????? Iterator<Proxy> it = sel.select(uri).iterator();
??????????????????? while (it.hasNext()) {
??????????????????????? p = it.next();
??????????????????????? try {
??????????????????????????? if (!failedOnce) {
??????????????????????????????? http = getNewHttpClient(url, p, connectTimeout);
...
}
?
getNewHttpClient(){
...
??????? return HttpClient.New(url, p, connectTimeout, useCache);
...
sun.net.www.http.HttpClient.java的openServer()方法建立socket连接：
?
??? protected synchronized void openServer() throws IOException {
??????????? ...
??????????? if ((proxy != null) && (proxy.type() == Proxy.Type.HTTP)) {
??????????????? sun.net.www.URLConnection.setProxiedHost(host);
??????????????? if (security != null) {
??????????????????? security.checkConnect(host, port);
??????????????? }
??????????????? privilegedOpenServer((InetSocketAddress) proxy.address());最终socket连接的是设置的代理服务器的地址，
??????????? ...
}
?
??? private synchronized void privilegedOpenServer(final InetSocketAddress server)
???????? throws IOException
??? {
??????? try {
??????????? java.security.AccessController.doPrivileged(
??????????????? new java.security.PrivilegedExceptionAction() {
??????????????? public Object run() throws IOException {
??????????????????? openServer(server.getHostName(), server.getPort());? 注意openserver函数? 这里的server的getHostName是设置的代理服务器，（ip或者hostname，如果是host绑定设置的代理服务器的ip，那么这里getHostName出来的就是ip地址，可以去查看InetSocketAddress类的getHostName方法）
??????????????????? return null;
??????????????? }
??????????? });
??????? } catch (java.security.PrivilegedActionException pae) {
??????????? throw (IOException) pae.getException();
??????? }
??? }
?
?? public void openServer(String server, int port) throws IOException {
??????? serverSocket = doConnect(server, port);? 生成的Socket连接对象
??????? try {
??????????? serverOutput = new PrintStream(
??????????????? new BufferedOutputStream(serverSocket.getOutputStream()),
???????????????????????????????????????? false, encoding);?? 生成输出流，
??????? } catch (UnsupportedEncodingException e) {
??????????? throw new InternalError(encoding+" encoding not found");
??????? }
??????? serverSocket.setTcpNoDelay(true);
protected Socket doConnect (String server, int port)
??? throws IOException, UnknownHostException {
??????? Socket s;
??????? if (proxy != null) {
??????????? if (proxy.type() == Proxy.Type.SOCKS) {
??????????????? s = (Socket) AccessController.doPrivileged(
?????????????????????????????? new PrivilegedAction() {
?????????????????????????????????? public Object run() {
?????????????????????????????????????? return new Socket(proxy);
?????????????????????????????????? }});
??????????? } else
??????????????? s = new Socket(Proxy.NO_PROXY);
??????? } else
??????????? s = new Socket();
??????? // Instance specific timeouts do have priority, that means
??????? // connectTimeout & readTimeout (-1 means not set)
??????? // Then global default timeouts
??????? // Then no timeout.
??????? if (connectTimeout >= 0) {
??????????? s.connect(new InetSocketAddress(server, port), connectTimeout);
??????? } else {
??????????? if (defaultConnectTimeout > 0) {
??????????????? s.connect(new InetSocketAddress(server, port), defaultConnectTimeout);//连接到代理服务器，看下面Socket类的connect方法代码
??????????? } else {
??????????????? s.connect(new InetSocketAddress(server, port));
??????????? }
??????? }
??????? if (readTimeout >= 0)
??????????? s.setSoTimeout(readTimeout);
??????? else if (defaultSoTimeout > 0) {
??????????? s.setSoTimeout(defaultSoTimeout);
??????? }
??????? return s;
}
?
上面的new InetSocketAddress(server, port)这里会涉及到java DNS cache的处理，
?
? ????public InetSocketAddress(String hostname, int port) {
??????? if (port < 0 || port > 0xFFFF) {
??????????? throw new IllegalArgumentException("port out of range:" + port);
??????? }
??????? if (hostname == null) {
??????????? throw new IllegalArgumentException("hostname can't be null");
??????? }
??????? try {
??????????? addr = InetAddress.getByName(hostname); ?//这里会有java DNS缓存的处理，先从缓存取hostname绑定的ip地址，如果取不到再通过OS的DNS cache机制去取，取不到再从DNS服务器上取。
??????? } catch(UnknownHostException e) {
????????? ??this.hostname = hostname;
??????????? addr = null;
??????? }
??????? this.port = port;
??? }
?
?
?
当然最终的Socket.java的connect方法
java.net.socket
? ??????????
???public void connect(SocketAddress endpoint, int timeout) throws IOException {
??????? if (endpoint == null)
???????????
????????if (timeout < 0)
????????? throw new IllegalArgumentException("connect: timeout can't be negative");
?
??????? if (isClosed())
??????????? throw new SocketException("Socket is closed");
?
??????? if (!oldImpl && isConnected())
??????????? throw new SocketException("already connected");
?
??????? if (!(endpoint instanceof InetSocketAddress))
??????????? throw new IllegalArgumentException("Unsupported address type");
?
??????? InetSocketAddress epoint = (InetSocketAddress) endpoint;
?
??????? SecurityManager security = System.getSecurityManager();
??????? if (security != null) {
??????????? if (epoint.isUnresolved())
??????????????? security.checkConnect(epoint.getHostName(),
????????????????????????????????????? epoint.getPort());
??????????? else
??????????????? security.checkConnect(epoint.getAddress().getHostAddress(),
????????????????????????????????????? epoint.getPort());
??????? }
??? ????if (!created)
??????????? createImpl(true);
??????? if (!oldImpl)
??????????? impl.connect(epoint, timeout);
??????? else if (timeout == 0) {
??????????? if (epoint.isUnresolved())? //如果没有设置SocketAddress的ip地址，则用域名去访问
??????????????? impl.connect(epoint.getAddress().getHostName(),
???????????????????????????? epoint.getPort());
??????????? else
??????????????? impl.connect(epoint.getAddress(), epoint.getPort());? 最终socket连接的是设置的SocketAddress的ip地址，
??????? } else
??????????? throw new UnsupportedOperationException("SocketImpl.connect(addr, timeout)");
??????? connected = true;
??????? /*
???????? * If the socket was not bound before the connect, it is now because
???????? * the kernel will have picked an ephemeral port & a local address
??????? ?*/
??????? bound = true;
??? }
?
我们再看下通过socket来发送HTTP请求的处理代码，也就是sun.net.www.protocl.http.HttpURLConnection.java的getInputStream方法中调用的writeRequests()方法：?
private void writeRequests() throws IOException {? 这段代码就是封装http请求的头请求信息，通过socket发送出去
??????? /* print all message headers in the MessageHeader
???????? * onto the wire - all the ones we've set and any
???????? * others that have been set
???????? */
??????? // send any pre-emptive authentication
??????? if (http.usingProxy) {
??????????? setPreemptiveProxyAuthentication(requests);
??????? }
??????? if (!setRequests) {
?
??????????? /* We're very particular about the order in which we
???????????? * set the request headers here.? The order should not
???????????? * matter, but some careless CGI programs have been
???????????? * written to expect a very particular order of the
???????????? * standard headers.? To name names, the order in which
???????????? * Navigator3.0 sends them.? In particular, we make *sure*
???????????? * to send Content-type: <> and Content-length:<> second
???????????? * to last and last, respectively, in the case of a POST
???????????? * request.
???????????? */
??????????? if (!failedOnce)
??????????????? requests.prepend(method + " " + http.getURLFile()+" "? +
???????????????????????????????? httpVersion, null);
??????????? if (!getUseCaches()) {
??????????????? requests.setIfNotSet ("Cache-Control", "no-cache");
??????????????? requests.setIfNotSet ("Pragma", "no-cache");
??????????? }
??????????? requests.setIfNotSet("User-Agent", userAgent);
??????????? int port = url.getPort();
??????????? String host = url.getHost();
??????????? if (port != -1 && port != url.getDefaultPort()) {
??????????????? host += ":" + String.valueOf(port);
??????????? }
??????????? requests.setIfNotSet("Host", host);
??????????? requests.setIfNotSet("Accept", acceptString);
?
??????????? /*
???????????? * For HTTP/1.1 the default behavior is to keep connections alive.
???????????? * However, we may be talking to a 1.0 server so we should set
???????????? * keep-alive just in case, except if we have encountered an error
???????????? * or if keep alive is disabled via a system property
???????????? */
?
??????????? // Try keep-alive only on first attempt
??????????? if (!failedOnce && http.getHttpKeepAliveSet()) {
??????????????? if (http.usingProxy) {
??????????????????? requests.setIfNotSet("Proxy-Connection", "keep-alive");
??????????????? } else {
??????????????????? requests.setIfNotSet("Connection", "keep-alive");
??????????????? }
??????????? } else {
??????????????? /*
???????????????? * RFC 2616 HTTP/1.1 section 14.10 says:
???????????????? * HTTP/1.1 applications that do not support persistent
???????????????? * connections MUST include the "close" connection option
???????????? ????* in every message
???????????????? */
??????????????? requests.setIfNotSet("Connection", "close");
??????????? }
??????????? // Set modified since if necessary
??????????? long modTime = getIfModifiedSince();
??????????? if (modTime != 0 ) {
??????????????? Date date = new Date(modTime);
??????????????? //use the preferred date format according to RFC 2068(HTTP1.1),
??????????????? // RFC 822 and RFC 1123
??????????????? SimpleDateFormat fo =
????????????????? new SimpleDateFormat ("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US);
??????????????? fo.setTimeZone(TimeZone.getTimeZone("GMT"));
??????????????? requests.setIfNotSet("If-Modified-Since", fo.format(date));
??????????? }
??????????? // check for preemptive authorization
??????????? AuthenticationInfo sauth = AuthenticationInfo.getServerAuth(url);
??????????? if (sauth != null && sauth.supportsPreemptiveAuthorization() ) {
??????????????? // Sets "Authorization"
??????????????? requests.setIfNotSet(sauth.getHeaderName(), sauth.getHeaderValue(url,method));
??????????????? currentServerCredentials = sauth;
??????????? }
?
??????????? if (!method.equals("PUT") && (poster != null || streaming())) {
??????????????? requests.setIfNotSet ("Content-type",
??????????????????????? "application/x-www-form-urlencoded");
??????????? }
?
??????????? if (streaming()) {
??????????????? if (chunkLength != -1) {
??????????????????? requests.set ("Transfer-Encoding", "chunked");
??????????????? } else {
??????????????????? requests.set ("Content-Length", String.valueOf(fixedContentLength));
??????????????? }
??????????? } else if (poster != null) {
??????????????? /* add Content-Length & POST/PUT data */
??????????????? synchronized (poster) {
??????????????????? /* close it, so no more data can be added */
??????????????????? poster.close();
??????????????????? requests.set("Content-Length",
???????????????????????????????? String.valueOf(poster.size()));
??????????????? }
??????????? }
?
??????????? // get applicable cookies based on the uri and request headers
??????????? // add them to the existing request headers
??????????? setCookieHeader();
…
}
?
?
再来看看把socket响应信息解析为http的响应信息的代码：
sun.net.www.http.HttpClient.java的parseHTTP方法：
private boolean parseHTTPHeader(MessageHeader responses, ProgressSource pi, HttpURLConnection httpuc)
??? throws IOException {
??????? /* If "HTTP/*" is found in the beginning, return true.? Let
???????? * HttpURLConnection parse the mime header itself.
???????? *
???????? * If this isn't valid HTTP, then we don't try to parse a header
???????? * out of the beginning of the response into the responses,
???????? * and instead just queue up the output stream to it's very beginning.
???????? * This seems most reasonable, and is what the NN browser does.
???????? */
?
??????? keepAliveConnections = -1;
??????? keepAliveTimeout = 0;
?
??????? boolean ret = false;
??????? byte[] b = new byte[8];
?
??????? try {
??????????? int nread = 0;
??????????? serverInput.mark(10);
??????????? while (nread < 8) {
??????????????? int r = serverInput.read(b, nread, 8 - nread);
??????????????? if (r < 0) {
??????????????????? break;
??????????????? }
??????????????? nread += r;
??????????? }
??????????? String keep=null;
??????????? ret = b[0] == 'H' && b[1] == 'T'
??????????????????? && b[2] == 'T' && b[3] == 'P' && b[4] == '/' &&
??????????????? b[5] == '1' && b[6] == '.';
??????????? serverInput.reset();
??????????? if (ret) { // is valid HTTP - response started w/ "HTTP/1."
??????????????? responses.parseHeader(serverInput);
?
??????????????? // we've finished parsing http headers
??????????????? // check if there are any applicable cookies to set (in cache)
??????????????? if (cookieHandler != null) {
??????????????????? URI uri = ParseUtil.toURI(url);
??????????????????? // NOTE: That cast from Map shouldn't be necessary but
??????????????????? // a bug in javac is triggered under certain circumstances
??????????????????? // So we do put the cast in as a workaround until
??????????????????? // it is resolved.
??????????????????? if (uri != null)
??????????????????????? cookieHandler.put(uri, (Map<java.lang.String,java.util.List<java.lang.String>>)responses.getHeaders());
??????????????? }
?
??????????????? /* decide if we're keeping alive:
???????????????? * This is a bit tricky.? There's a spec, but most current
???????????????? * servers (10/1/96) that support this differ in dialects.
???????????????? * If the server/client misunderstand each other, the
???????????????? * protocol should fall back onto HTTP/1.0, no keep-alive.
???????????????? */
??????????????? if (usingProxy) { // not likely a proxy will return this
??????????????????? keep = responses.findValue("Proxy-Connection");
??????????????? }
??????????????? if (keep == null) {
??????????????????? keep = responses.findValue("Connection");
??????????????? }
??????????????? if (keep != null && keep.toLowerCase().equals("keep-alive")) {
??????????????????? /* some servers, notably Apache1.1, send something like:
???????????????????? * "Keep-Alive: timeout=15, max=1" which we should respect.
???????????????????? */
??????????????????? HeaderParser p = new HeaderParser(
??????????????????????????? responses.findValue("Keep-Alive"));
??????????????????? if (p != null) {
??????????????????????? /* default should be larger in case of proxy */
??????????????????????? keepAliveConnections = p.findInt("max", usingProxy?50:5);
??????????????????????? keepAliveTimeout = p.findInt("timeout", usingProxy?60:5);
??????????????????? }
??????????????? } else if (b[7] != '0') {
??????????????????? /*
???????????????????? * We're talking 1.1 or later. Keep persistent until
???????????????????? * the server says to close.
???????????????????? */
??????????????????? if (keep != null) {
??????????????????????? /*
???????????????????????? * The only Connection token we understand is close.
???????????????????????? * Paranoia: if there is any Connection header then
???????????????????????? * treat as non-persistent.
???????????????????????? */
?????????????????????? ?keepAliveConnections = 1;
??????????????????? } else {
??????????????????????? keepAliveConnections = 5;
??????????????????? }
??????????????? }
……
}
?
?
对于java.net包的http，ftp等各种协议的底层实现，可以参考rt.jar下面的几个包的代码：
sun.net.www.protocl下的几个包。
?
?
?????????? ??? HostConfiguration conf = new HostConfiguration();
?????????? ??? conf.setHost(host);
?????????? ??? conf.setProxy(ip, 80);
public static void jdkDnsNoCache(final String host, final String ip)
?????????? throws SecurityException, NoSuchFieldException,
?????????? IllegalArgumentException, IllegalAccessException {
?????? if (StringUtils.isBlank(host)) {
?????????? return;
?????? }
?????? final Class clazz = java.net.InetAddress.class;
?????? final Field cacheField = clazz.getDeclaredField("addressCache");
?????? cacheField.setAccessible(true);
?????? final Object o = cacheField.get(clazz);
?????? Class clazz2 = o.getClass();
?????? final Field cacheMapField = clazz2.getDeclaredField("cache");
?????? cacheMapField.setAccessible(true);
?????? final Map cacheMap = (Map) cacheMapField.get(o);
?????? AccessController.doPrivileged(new PrivilegedAction() {
?????????? public Object run() {
????????????? try {
????????????????? synchronized (o) {// 同步是必须的,因为o可能会有多个线程同时访问修改。
???????????????????? // cacheMap.clear();//这步比较关键，用于清除原来的缓存
//?????????????????? cacheMap.remove(host);
???????????????????? if (!StringUtils.isBlank(ip)) {
???????????????????????? InetAddress inet = InetAddress.getByAddress(host,IPUtil.int2byte(ip));
???????????????????????? InetAddress addressstart = InetAddress.getByName(host);
???????????????????????? Object cacheEntry = cacheMap.get(host);
???????????????????????? cacheMap.put(host,newCacheEntry(inet,cacheEntry));
//?????????????????????? cacheMap.put(host,newCacheEntry(newInetAddress(host, ip)));
???????????????????? }else{
???????????????????????? cacheMap.remove(host);
???????????????????? }
//?????????????????? System.out.println(getStaticProperty(
//????????????????????????? "java.net.InetAddress", "addressCacheInit"));
???????????????????? // System.out.println(invokeStaticMethod("java.net.InetAddress","getCachedAddress",new
???????????????????? // Object[]{host}));
????????????????? }
????????????? } catch (Throwable te) {
????????????????? throw new RuntimeException(te);
????????????? }
????????????? return null;
?????????? }
?????? });
?????? final Map cacheMapafter = (Map) cacheMapField.get(o);
?????? System.out.println(cacheMapafter);
?
1.在${java_home}/jre/lib/secuiry/java.secuiry文件，修改下面为?
??networkaddress.cache.negative.ttl=0?? DNS解析不成功的缓存时间
networkaddress.cache.ttl=0??? DNS解析成功的缓存的时间
2.jvm启动时增加下面两个启动环境变量
? -Dsun.net.inetaddr.ttl=0
?
如果在java程序中使用，可以这么设置设置：
?? ?java.security.Security.setProperty("networkaddress.cache.ttl" , "0");
http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6247501
?
/etc/resolve.conf
/etc/nscd.conf
/etc/nsswitch.conf
?
http://www.linuxfly.org/post/543/
http://linux.die.net/man/5/nscd.conf
http://www.linuxhomenetworking.com/wiki/index.php/Quick_HOWTO_:_Ch18_:_Configuring_DNS
http://linux.die.net/man/5/nscd.conf