From be686ee021a61d26406b930a3ed35ed04ef231ed Mon Sep 17 00:00:00 2001 From: Danilo Tuler Date: Tue, 10 Dec 2013 01:44:28 -0200 Subject: [PATCH 001/105] adding eclipse files to .gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1508b92..a9aaddb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ .idea/ +.project +.classpath +.settings/ target/ **/target/ -**/.idea/ \ No newline at end of file +**/.idea/ From 0b8af9df7a3119647e9bc7ae4626af22146894d0 Mon Sep 17 00:00:00 2001 From: Danilo Tuler Date: Tue, 10 Dec 2013 01:47:27 -0200 Subject: [PATCH 002/105] checking if the user-agent header contains the string defined in the crawler user-agents collection --- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 5e9defd..8a63e24 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -212,7 +212,7 @@ private boolean isInSearchUserAgent(final String useAgent) { return from(getCrawlerUserAgents()).anyMatch(new Predicate() { @Override public boolean apply(String item) { - return item.equalsIgnoreCase(useAgent); + return useAgent.toLowerCase().indexOf(item.toLowerCase()) >= 0; } }); } From 049d2a4cde37f9dd56657f8b590a35239744bc68 Mon Sep 17 00:00:00 2001 From: Danilo Tuler Date: Tue, 10 Dec 2013 01:48:50 -0200 Subject: [PATCH 003/105] variable typo: useAgent -> userAgent --- .../com/github/greengerong/PreRenderSEOFilter.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 8a63e24..adc358e 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -128,7 +128,7 @@ private List getBlacklist() { } private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { - final String useAgent = request.getHeader("User-Agent"); + final String userAgent = request.getHeader("User-Agent"); final String url = request.getRequestURL().toString(); final String referer = request.getHeader("Referer"); @@ -136,11 +136,11 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI return true; } - if (StringUtils.isBlank(useAgent)) { + if (StringUtils.isBlank(userAgent)) { return false; } - if (!isInSearchUserAgent(useAgent)) { + if (!isInSearchUserAgent(userAgent)) { return false; } @@ -208,11 +208,11 @@ public boolean apply(String item) { }); } - private boolean isInSearchUserAgent(final String useAgent) { + private boolean isInSearchUserAgent(final String userAgent) { return from(getCrawlerUserAgents()).anyMatch(new Predicate() { @Override public boolean apply(String item) { - return useAgent.toLowerCase().indexOf(item.toLowerCase()) >= 0; + return userAgent.toLowerCase().indexOf(item.toLowerCase()) >= 0; } }); } From a313a0a9b20f20174898d237cc23413f412d78ee Mon Sep 17 00:00:00 2001 From: Danilo Tuler Date: Tue, 10 Dec 2013 01:56:31 -0200 Subject: [PATCH 004/105] only intercept GET requests --- .../com/github/greengerong/PreRenderSEOFilter.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index adc358e..47f9fa0 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -3,6 +3,7 @@ import com.google.common.base.Predicate; import com.google.common.collect.Lists; + import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.methods.GetMethod; @@ -10,6 +11,7 @@ import javax.servlet.*; import javax.servlet.http.HttpServletRequest; + import java.io.IOException; import java.io.PrintWriter; import java.net.URISyntaxException; @@ -132,20 +134,28 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI final String url = request.getRequestURL().toString(); final String referer = request.getHeader("Referer"); + if (!"GET".equals(request.getMethod())) { + // only respond to GET requests + return false; + } + if (hasEscapedFragment(request)) { + // request has the escape fragment, as defined by google, intercept the request return true; } if (StringUtils.isBlank(userAgent)) { + // no User-Agent header, don't intercept return false; } if (!isInSearchUserAgent(userAgent)) { + // User-Agent is not a search bot, don't intercept return false; } - if (isInResources(url)) { + // request is for a (static) resource, don't intercept return false; } From a6b237b2c93e35c332b7f50601ca3af8bf94e882 Mon Sep 17 00:00:00 2001 From: Danilo Tuler Date: Tue, 10 Dec 2013 02:42:42 -0200 Subject: [PATCH 005/105] migrating to HttpClient 4 --- pom.xml | 18 ++-- .../greengerong/PreRenderSEOFilter.java | 84 ++++++++++++------- 2 files changed, 66 insertions(+), 36 deletions(-) diff --git a/pom.xml b/pom.xml index f506089..8446a79 100644 --- a/pom.xml +++ b/pom.xml @@ -32,9 +32,10 @@ 4.11 1.9.5 15.0 - 2.6 + 2.6 + 2.4 2.5 - 3.1 + 4.3.1 1.2.9 @@ -65,7 +66,12 @@ commons-lang commons-lang - ${commons-lange.version} + ${commons-lang.version} + + + commons-io + commons-io + ${commons-io.version} javax.servlet @@ -74,9 +80,9 @@ provided - commons-httpclient - commons-httpclient - ${commons-httpclient.version} + org.apache.httpcomponents + httpclient + ${httpclient.version} diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 47f9fa0..b5430f2 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,16 +1,7 @@ package com.github.greengerong; -import com.google.common.base.Predicate; -import com.google.common.collect.Lists; - -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpMethod; -import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.commons.lang.StringUtils; - -import javax.servlet.*; -import javax.servlet.http.HttpServletRequest; +import static com.google.common.collect.FluentIterable.from; import java.io.IOException; import java.io.PrintWriter; @@ -19,15 +10,49 @@ import java.util.List; import java.util.regex.Pattern; -import static com.google.common.collect.FluentIterable.from; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpHost; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.DefaultProxyRoutePlanner; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; + +import com.google.common.base.Predicate; +import com.google.common.collect.Lists; public class PreRenderSEOFilter implements Filter { private FilterConfig filterConfig; + + private CloseableHttpClient httpClient; @Override public void init(FilterConfig filterConfig) throws ServletException { this.filterConfig = filterConfig; + + HttpClientBuilder builder = HttpClients.custom(); + + final String proxy = filterConfig.getInitParameter("proxy"); + if (StringUtils.isNotBlank(proxy)) { + final int proxyPort = Integer.parseInt(filterConfig.getInitParameter("proxyPort")); + DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); + builder = builder.setRoutePlanner(routePlanner); + } + + builder = builder.setConnectionManager(new PoolingHttpClientConnectionManager()); + httpClient = builder.build(); } @Override @@ -51,14 +76,16 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo private ResponseResult getPrerenderedPageResponse(HttpServletRequest request) throws IOException { final String apiUrl = getApiUrl(getFullUrl(request)); - final HttpClient httpClient = new HttpClient(); - final GetMethod getMethod = new GetMethod(apiUrl); - setConfig(httpClient); + final HttpGet getMethod = new HttpGet(apiUrl); setHttpHeader(getMethod); - final int code = httpClient.executeMethod(getMethod); - final String responseBody = getMethod.getResponseBodyAsString(); - String body = new String(responseBody.getBytes("utf-8")); - return new ResponseResult(code, body); + CloseableHttpResponse response = httpClient.execute(getMethod); + try { + final int code = response.getStatusLine().getStatusCode(); + String body = IOUtils.toString(response.getEntity().getContent(), "utf-8"); + return new ResponseResult(code, body); + } finally { + response.close(); + } } private String getFullUrl(HttpServletRequest request) { @@ -71,22 +98,19 @@ private String getFullUrl(HttpServletRequest request) { return url.toString(); } - private void setHttpHeader(HttpMethod httpMethod) { - httpMethod.setRequestHeader("Cache-Control", "no-cache"); - httpMethod.setRequestHeader("Content-Type", "text/html"); - } - - private void setConfig(HttpClient httpClient) { - final String proxy = filterConfig.getInitParameter("proxy"); - if (StringUtils.isNotBlank(proxy)) { - final int proxyPort = Integer.parseInt(filterConfig.getInitParameter("proxyPort")); - httpClient.getHostConfiguration().setProxy(proxy, proxyPort); - } + private void setHttpHeader(HttpGet httpMethod) { + httpMethod.setHeader("Cache-Control", "no-cache"); + httpMethod.setHeader("Content-Type", "text/html"); } @Override public void destroy() { filterConfig = null; + try { + httpClient.close(); + } catch (IOException e) { + e.printStackTrace(); + } } private List getCrawlerUserAgents() { @@ -134,7 +158,7 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI final String url = request.getRequestURL().toString(); final String referer = request.getHeader("Referer"); - if (!"GET".equals(request.getMethod())) { + if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { // only respond to GET requests return false; } From e9bbf5f526e41dcc9f32668c718caa744cea27b7 Mon Sep 17 00:00:00 2001 From: Danilo Tuler Date: Tue, 10 Dec 2013 03:05:32 -0200 Subject: [PATCH 006/105] proper handling of request and response headers --- .../greengerong/PreRenderSEOFilter.java | 142 +++++++++++++----- 1 file changed, 103 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index b5430f2..be6b4db 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -3,10 +3,13 @@ import static com.google.common.collect.FluentIterable.from; +import java.io.Closeable; import java.io.IOException; -import java.io.PrintWriter; +import java.io.OutputStream; +import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; +import java.util.Enumeration; import java.util.List; import java.util.regex.Pattern; @@ -17,17 +20,25 @@ import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIUtils; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.DefaultProxyRoutePlanner; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicHeader; +import org.apache.http.message.HeaderGroup; import com.google.common.base.Predicate; import com.google.common.collect.Lists; @@ -60,34 +71,109 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo throws IOException, ServletException { try { final HttpServletRequest request = (HttpServletRequest) servletRequest; + final HttpServletResponse response = (HttpServletResponse) servletResponse; if (shouldShowPrerenderedPage(request)) { - final ResponseResult result = getPrerenderedPageResponse(request); - if (result.getStatusCode() == 200) { - final PrintWriter writer = servletResponse.getWriter(); - writer.write(result.getResponseBody()); - writer.flush(); - return; - } + if (proxyPrerenderedPageResponse(request, response)) { + return; + } } } catch (Exception e) { } filterChain.doFilter(servletRequest, servletResponse); } - private ResponseResult getPrerenderedPageResponse(HttpServletRequest request) throws IOException { + private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException { final String apiUrl = getApiUrl(getFullUrl(request)); final HttpGet getMethod = new HttpGet(apiUrl); - setHttpHeader(getMethod); - CloseableHttpResponse response = httpClient.execute(getMethod); + copyRequestHeaders(request, getMethod); + CloseableHttpResponse httpResponse = httpClient.execute(getMethod); try { - final int code = response.getStatusLine().getStatusCode(); - String body = IOUtils.toString(response.getEntity().getContent(), "utf-8"); - return new ResponseResult(code, body); + if (httpResponse.getStatusLine().getStatusCode() == 200) { + copyResponseHeaders(httpResponse, response); + copyResponseEntity(httpResponse, response); + return true; + } } finally { - response.close(); + httpResponse.close(); } + return false; + } + + /** Copy proxied response headers back to the servlet client. */ + protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { + for (Header header : proxyResponse.getAllHeaders()) { + if (hopByHopHeaders.containsHeader(header.getName())) + continue; + servletResponse.addHeader(header.getName(), header.getValue()); + } } - + + /** Copy response body data (the entity) from the proxy to the servlet client. */ + protected void copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) throws IOException { + HttpEntity entity = proxyResponse.getEntity(); + if (entity != null) { + OutputStream servletOutputStream = servletResponse.getOutputStream(); + try { + entity.writeTo(servletOutputStream); + } finally { + closeQuietly(servletOutputStream); + } + } + } + + protected void closeQuietly(Closeable closeable) { + try { + closeable.close(); + } catch (IOException e) { + } + } + + /** These are the "hop-by-hop" headers that should not be copied. + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html + * I use an HttpClient HeaderGroup class instead of Set because this + * approach does case insensitive lookup faster. + */ + protected static final HeaderGroup hopByHopHeaders; + static { + hopByHopHeaders = new HeaderGroup(); + String[] headers = new String[] { + "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", + "TE", "Trailers", "Transfer-Encoding", "Upgrade" }; + for (String header : headers) { + hopByHopHeaders.addHeader(new BasicHeader(header, null)); + } + } + + /** Copy request headers from the servlet client to the proxy request. + * @throws URISyntaxException */ + protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) throws URISyntaxException { + // Get an Enumeration of all of the header names sent by the client + Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); + while (enumerationOfHeaderNames.hasMoreElements()) { + String headerName = (String) enumerationOfHeaderNames.nextElement(); + //Instead the content-length is effectively set via InputStreamEntity + if (headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH)) + continue; + if (hopByHopHeaders.containsHeader(headerName)) + continue; + + Enumeration headers = servletRequest.getHeaders(headerName); + while (headers.hasMoreElements()) {//sometimes more than one value + String headerValue = (String) headers.nextElement(); + // In case the proxy host is running multiple virtual servers, + // rewrite the Host header to ensure that we get content from + // the correct virtual server + if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { + HttpHost host = URIUtils.extractHost(new URI(getPrerenderServiceUrl())); + headerValue = host.getHostName(); + if (host.getPort() != -1) + headerValue += ":"+host.getPort(); + } + proxyRequest.addHeader(headerName, headerValue); + } + } + } + private String getFullUrl(HttpServletRequest request) { final StringBuffer url = request.getRequestURL(); final String queryString = request.getQueryString(); @@ -98,11 +184,6 @@ private String getFullUrl(HttpServletRequest request) { return url.toString(); } - private void setHttpHeader(HttpGet httpMethod) { - httpMethod.setHeader("Cache-Control", "no-cache"); - httpMethod.setHeader("Content-Type", "text/html"); - } - @Override public void destroy() { filterConfig = null; @@ -251,21 +332,4 @@ public boolean apply(String item) { }); } - private class ResponseResult { - private int statusCode; - private String responseBody; - - public ResponseResult(int code, String body) { - statusCode = code; - responseBody = body; - } - - private int getStatusCode() { - return statusCode; - } - - private String getResponseBody() { - return responseBody; - } - } } From d98e70e94d2655a23cbcbe8dee0db68649212f9a Mon Sep 17 00:00:00 2001 From: greengerongg Date: Mon, 16 Dec 2013 20:19:36 +0800 Subject: [PATCH 007/105] =?UTF-8?q?=C3=AFsssue=20for=20jboss?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .DS_Store | Bin 0 -> 6148 bytes demo/.DS_Store | Bin 0 -> 6148 bytes demo/demo.iml | 1 - demo/src/.DS_Store | Bin 0 -> 6148 bytes demo/src/main/.DS_Store | Bin 0 -> 6148 bytes demo/src/main/webapp/.DS_Store | Bin 0 -> 6148 bytes pom.xml | 1 + src/.DS_Store | Bin 0 -> 6148 bytes src/main/.DS_Store | Bin 0 -> 6148 bytes src/main/java/.DS_Store | Bin 0 -> 6148 bytes src/main/java/com/.DS_Store | Bin 0 -> 6148 bytes src/main/java/com/github/.DS_Store | Bin 0 -> 6148 bytes 12 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 .DS_Store create mode 100644 demo/.DS_Store create mode 100644 demo/src/.DS_Store create mode 100644 demo/src/main/.DS_Store create mode 100644 demo/src/main/webapp/.DS_Store create mode 100644 src/.DS_Store create mode 100644 src/main/.DS_Store create mode 100644 src/main/java/.DS_Store create mode 100644 src/main/java/com/.DS_Store create mode 100644 src/main/java/com/github/.DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..344f67ceab9667324694eeaece27fa3dda580ccd GIT binary patch literal 6148 zcmeHK%}&BV5T1pqZPbK|3CE3RVn9VWcxVVF9!((OAVynDgrrhJiyRPbOne^S#^-Tn zcOfEr;9~rlNoK#<*_m$UYqQe@08twSC4d|N*r}k3plA5wc;JVT zkMozMXmc5I_fRA$tG*{6W}@lVb5zWU|)U`iAW|_Ex*q>j%-e7LEEreBS6r z<3=y&_}=hB*@>5o;+i*9D`#HP#T8{YqbNkXZK7MfAn9J_@{`Gaxm=K?qw2IECzVRE zAP*0!)2U+1)!nZF=|9hsRWNC-5^qnQC_;2$tR=YvEgbPc8&)zN{2x&k25Z=@2m z=`KMz(x7WF)rc!7LZ>3?RAH_dLZ_o&nmE^Bs!^wdP&?yu%+A8xP=wkY{ZfU4a5eJC z3@`(W3}p2*NB#fg>-+y=63>_cX5ddTAgp?$Uc-`1Z(S;mdaXpgLM5TNRO37a4b_S< fmRfNhRSEhf8HldIR3my&_(wp|zymYztqi;YyJ3Ir literal 0 HcmV?d00001 diff --git a/demo/.DS_Store b/demo/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..d0ec3957f494e138754c9af45d421508531d9179 GIT binary patch literal 6148 zcmeH~&r8EF6vq=cHARJ9gdKzDA)7k4gNITkcoaI=!HKq3T%ok1?c8Bji{Rhmf8*cd zdr1mTcNUQsNWL$5FG=%(<_BYpyYsNin9UdykWo1rYZ5lEb#0hnRE}fhvmj0bsJ#a> z%&)v@y6~d#KJc?^ zWfy)i&s{%Lo=d-&f=Ah-G>yTYnCNj77SkKMwOk(cdTrS~9h7ak?Dsoud2&1`OG^mL zeiQo#hi4Zf|EYX_sWe3nkX70aN^8F}Usy z?b5_KMk}qlJL76*7{|<9KVG<+9onS|XWY?BB@rM3s{|UlYUBBT4!_LWNB(LTsu2Mq z@XrYF=Fl6uuqbo3{#YKKwG#3zWK0-Wp#cf(l}i9ra35(Yr}j& - diff --git a/demo/src/.DS_Store b/demo/src/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..e66c6b5437e4c9df30bd8a4979ba83365488deeb GIT binary patch literal 6148 zcmeH~K}*9h6vvaTYKscJ2s;MPMK*PA2M?u8@F;Y!gA;A3xI%5m+PTB57QxTsxAF7% zUXqH_okipYlK)HIdr9+yEeT_cd-JHnn9Ue7p`vm$))X9G>uTs=RE~3mi+ma;SqS|% z?fdJ3-)^vb7O#ZfW2wp+{PVYlmu&Pl&?#Io0GJL34LUzV0> zTK22Cw|{VYJ`5hqr{_v>)`ghzk? literal 0 HcmV?d00001 diff --git a/demo/src/main/.DS_Store b/demo/src/main/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..496035a1af584566e79dc9088a9c3c0ee69cbcd5 GIT binary patch literal 6148 zcmeHK%}T>S5Z-NTO;8~hp~uB@kyfqk!9yq&JPIlFU_}#BY#=0|NzI{FgW&V{Ha?Ft zyPHEWXAwICv)}Ce>}Eg6{xHV)c$NivHzj+ZCJPu-6qA=cDb` zGz`=FFpf`7uWqL9%l7rHQ2cSATFQ~e65he6Eall*MPVF0fxoONa|ww7Vt^PR26mVM zeJB_QJInzsofsen>KVZOL4YFK8ViMT>wpHY&lqnZqJWKW2}GgM)>tTn2nbiDfU1=1 zCk9vL;1@d2)>tT1<&4XjVH`bk`FP=ScJK?C&bY0RT4I10s4~!0T?^0uYxreqANkc1 z>JbCPz&~SvwyPE(St~#vK~XTTKm!Eq$|V3gxQ}#XP{jq>kY{Tw6yhl8 Rm*s$T5s-vXLk#=^1K(ORNofE8 literal 0 HcmV?d00001 diff --git a/demo/src/main/webapp/.DS_Store b/demo/src/main/webapp/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..2a74ced11ee55ebb91e09228cecc47288de92ca0 GIT binary patch literal 6148 zcmeHK%}T>S5T3Qwrl^pM(BtB{N~_lP;30%o^ddqFJy_9%hz*3MG^st*Y7l%L-^S;0 zW_Jr#>s7?g!0tCYKf9R^vOfSI+;LC^umQlrMkq*GB4l3bS~J0fLd|gw7jWuNlA#|h zndmQ?=-Ug>g%-H52cNz__R;g^52G+iE0uS#mdmei6s%&=+Uj+~K@d;ecsvM_t9C!0 zw8P-e_eR%BPrY=UxZX&4F1@sm9;J8UI6}Q=qPt;`_HXRcbb3&$m1Xs~F)PbyyyvNzobEX*-NVbw~gL8O6LHlM(7yKG^(uwJGwrTKTk-4I=v+br9sDFrV%|T zLZ>3?RAHVNLZ_o$nmET`rctMZP%Gm+X65$=PMPZV!OpgLU9=>D5x)70&qb4NJ%-hUy_b- Yj=@YL%|dpSj>s1QMF@Ayz%MZH0p1};X#fBK literal 0 HcmV?d00001 diff --git a/pom.xml b/pom.xml index f506089..7d5ee32 100644 --- a/pom.xml +++ b/pom.xml @@ -49,6 +49,7 @@ com.google.guava guava ${guava.version} + cdi1.0 junit diff --git a/src/.DS_Store b/src/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..e66c6b5437e4c9df30bd8a4979ba83365488deeb GIT binary patch literal 6148 zcmeH~K}*9h6vvaTYKscJ2s;MPMK*PA2M?u8@F;Y!gA;A3xI%5m+PTB57QxTsxAF7% zUXqH_okipYlK)HIdr9+yEeT_cd-JHnn9Ue7p`vm$))X9G>uTs=RE~3mi+ma;SqS|% z?fdJ3-)^vb7O#ZfW2wp+{PVYlmu&Pl&?#Io0GJL34LUzV0> zTK22Cw|{VYJ`5hqr{_v>)`ghzk? literal 0 HcmV?d00001 diff --git a/src/main/.DS_Store b/src/main/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..1310f12577816836235239bd947be7ffa0766e36 GIT binary patch literal 6148 zcmeH~K}*9h6vvaTYKscJ2s;MPLpF782M?u8@F;Y!gA;A3xJqfq+HHreS_D6j-^S15 zdr2xzcNUQsNd7N*?V;Vgo?`1*idkIqpPNaQ8~^LEV4-urvdce zwC}GAe!IaQna}3T|N8yGB#yGY)q3Z(y0NvrV_KHEHyTDsm@eFOmW0{0H%=E`6y68^ z^hVmbpU*PapGwb_pO3*K?LnHxV2^b4FbebWt=(8Ik2{^FXrJ|prdW2nt)@6V=@o@# znwI@)9vmK>Uk?0L@$_6OjvO${vahw07cf4zG54lPoF)%2R>fCwq(p!S5CI}U1ggWq z+$9!7fCvzQZv^o2D4?P`T1%z!UI`6epV4oiVgVc95~xC}qqS5D9w=O;f~r)mpBP-F z!?@6Kj@DACN@rZI4D;xftH%qMtHZdE;fy;yPc>Su3F4LB)b`B^pq`Ub_T92mePJ(y9CcZHRNUmP&CJ9GB^! ObPSjXs2s=0rePvOsJ(|% zP`Ha52`+qQPbqd1M^!jtnf$}athT=;Qx7Y4H{ zW#>UY&%9u!JQqPe0gtkWQYK)JP4p;^^2xQ+TrLm0-InN_^oy2Q_ImA>I6mqZg>6~3 z^JeYsADo^KgQw#8rPLHTKvrqjXeF;;m_hZ$W?9eV$IODEXDv1CQSS3)`RR_=iGx%lJKJr()P>l!> zfqzDTHwON|gGHIM^~dt?tQC-NAY;O~5)DXTuUrD4g8N8QIkjJ)4RNl~QY(&v?W#K{ OT?7;mDu}=@5cmYR6Gp25 literal 0 HcmV?d00001 diff --git a/src/main/java/com/.DS_Store b/src/main/java/com/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..e0512af466f9b9e96547f2cda22eb0628ed9b252 GIT binary patch literal 6148 zcmeHK%}T>S5Z-NTO;8~hp~uB@kyfqk!9xfYJPIlFU_}!mO`tTTN$jCUgW&V{Ha?Ft zyPHEWXAwICv)}Ce>}Eg6{xHV4lLdXoEXJ4tipbHZ5p>sv8af$~%P}(M`}Fy>FPNaX6bs%OKu3akdPSJ2#9sZWKKE z-uhnJsh4Jn@n)=NW(NP7~;3$SN8dm064_+WLm+q1!-EBcqCyeqcDVXrGL&PRD} z8ir|o7{@24S2q*yC4YS@6n`A3mU5)Agm*A1OL=zJ%SF7LgTJgPa|ww7Vt^PR26mVM zeJB_QJInzsofsen>KVZOL4YFK8ViMT>wpHY&lqnZqJWKW2}GgM)>tTn2nbiDfU1=1 zCk9vL;1@d2)>tT1<&4XjVH`bk`FP=ScJK?C&bY0RT4I10s4~!0T?^0uYxreqANkc1 z>JbCPz&~Svx5n<+fko-F^~dt?tQDY-peUGEpaBAQ%6b|%Zb3AY~)C*$|BHp}d7^GRV`OZt_%FgbdVVcJNa1ca( zJh$Un7VzV$ta|8{+J=o9Cusn6+2q@aXvBs_#DLPtS#7lEX(Qr5V#Ntt0 b2bBVTfd-(Zu}}ye5c(02G|)f{{3-( Date: Mon, 16 Dec 2013 10:22:25 -0200 Subject: [PATCH 008/105] adding jetty plugin so we can run the web app easily with 'mvn jetty:run' --- demo/pom.xml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/demo/pom.xml b/demo/pom.xml index 6864939..b79f94a 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,10 +11,16 @@ com.github.greengerong prerender-java - 1.0-SNAPSHOT + 1.1-SNAPSHOT demo + + + org.eclipse.jetty + jetty-maven-plugin + + From 80d85b53205bd2e93d73dc8cf65e7702bcab0d5d Mon Sep 17 00:00:00 2001 From: Danilo Tuler Date: Mon, 16 Dec 2013 10:22:44 -0200 Subject: [PATCH 009/105] adding angularjs from cloud url --- demo/src/main/webapp/index.jsp | 189 +++------------------------------ 1 file changed, 15 insertions(+), 174 deletions(-) diff --git a/demo/src/main/webapp/index.jsp b/demo/src/main/webapp/index.jsp index b1e3b13..63d1612 100644 --- a/demo/src/main/webapp/index.jsp +++ b/demo/src/main/webapp/index.jsp @@ -1,176 +1,17 @@ - - - - -

Prerender java demo page

- -

use angularjs ng-repeat

-
    -
  • {{item}}
    • -
- - + + + + +

Prerender java demo page

+

use angularjs ng-repeat

+
    +
  • {{item}}
    • +
+ + From bd6e5212e47d6e6992445913f98e31e34d5ba2b0 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Mon, 16 Dec 2013 20:23:31 +0800 Subject: [PATCH 010/105] [maven-release-plugin] prepare release prerender-java-1.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7d5ee32..8641c03 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.1-SNAPSHOT + 1.1 jar prerender-java From 42743423491759b7380e92d30be6a2d64f72a95f Mon Sep 17 00:00:00 2001 From: greengerongg Date: Mon, 16 Dec 2013 20:23:43 +0800 Subject: [PATCH 011/105] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8641c03..1263f8c 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.1 + 1.2-SNAPSHOT jar prerender-java From 9322e6388e622bba35a3cec7ecda8146804343c6 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Mon, 16 Dec 2013 20:26:02 +0800 Subject: [PATCH 012/105] release --- demo/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/pom.xml b/demo/pom.xml index 6864939..92e02cf 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -4,7 +4,7 @@ com.github.greengerong demo war - 1.0 + 1.2-SNAPSHOT demo Maven Webapp http://maven.apache.org From 70c5b74172ad9f4f963e2bf0eb3adc8edaf669b6 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Mon, 16 Dec 2013 20:30:24 +0800 Subject: [PATCH 013/105] read me:change release version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ec66ca..9904e47 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.0 + 1.1 ## How it works From bedc861c01b11019d76c777a9b48fb0b4d462633 Mon Sep 17 00:00:00 2001 From: Danilo Tuler Date: Mon, 16 Dec 2013 10:39:48 -0200 Subject: [PATCH 014/105] removing tabs --- .../greengerong/PreRenderSEOFilter.java | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index be6b4db..81d3c5f 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -94,38 +94,38 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer return true; } } finally { - httpResponse.close(); + httpResponse.close(); } - return false; - } + return false; + } /** Copy proxied response headers back to the servlet client. */ protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { - for (Header header : proxyResponse.getAllHeaders()) { - if (hopByHopHeaders.containsHeader(header.getName())) - continue; - servletResponse.addHeader(header.getName(), header.getValue()); - } + for (Header header : proxyResponse.getAllHeaders()) { + if (hopByHopHeaders.containsHeader(header.getName())) + continue; + servletResponse.addHeader(header.getName(), header.getValue()); + } } /** Copy response body data (the entity) from the proxy to the servlet client. */ protected void copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) throws IOException { - HttpEntity entity = proxyResponse.getEntity(); - if (entity != null) { - OutputStream servletOutputStream = servletResponse.getOutputStream(); - try { - entity.writeTo(servletOutputStream); - } finally { - closeQuietly(servletOutputStream); - } - } + HttpEntity entity = proxyResponse.getEntity(); + if (entity != null) { + OutputStream servletOutputStream = servletResponse.getOutputStream(); + try { + entity.writeTo(servletOutputStream); + } finally { + closeQuietly(servletOutputStream); + } + } } protected void closeQuietly(Closeable closeable) { - try { - closeable.close(); - } catch (IOException e) { - } + try { + closeable.close(); + } catch (IOException e) { + } } /** These are the "hop-by-hop" headers that should not be copied. @@ -188,10 +188,10 @@ private String getFullUrl(HttpServletRequest request) { public void destroy() { filterConfig = null; try { - httpClient.close(); - } catch (IOException e) { - e.printStackTrace(); - } + httpClient.close(); + } catch (IOException e) { + e.printStackTrace(); + } } private List getCrawlerUserAgents() { @@ -240,27 +240,27 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI final String referer = request.getHeader("Referer"); if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { - // only respond to GET requests - return false; + // only respond to GET requests + return false; } if (hasEscapedFragment(request)) { - // request has the escape fragment, as defined by google, intercept the request + // request has the escape fragment, as defined by google, intercept the request return true; } if (StringUtils.isBlank(userAgent)) { - // no User-Agent header, don't intercept + // no User-Agent header, don't intercept return false; } if (!isInSearchUserAgent(userAgent)) { - // User-Agent is not a search bot, don't intercept + // User-Agent is not a search bot, don't intercept return false; } if (isInResources(url)) { - // request is for a (static) resource, don't intercept + // request is for a (static) resource, don't intercept return false; } From f5d0dcd202f376591bd662d78511211326439685 Mon Sep 17 00:00:00 2001 From: Danilo Tuler Date: Mon, 16 Dec 2013 13:08:22 -0200 Subject: [PATCH 015/105] replacing "me" example, as "me" is in the word Chrome, and may confuse the user --- demo/src/main/webapp/WEB-INF/web.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/src/main/webapp/WEB-INF/web.xml b/demo/src/main/webapp/WEB-INF/web.xml index 89ebe63..eadd7d5 100644 --- a/demo/src/main/webapp/WEB-INF/web.xml +++ b/demo/src/main/webapp/WEB-INF/web.xml @@ -13,7 +13,7 @@ crawlerUserAgents - me + YahooSeeker From 23905a1a184729fd12bc7b2056841054012addf8 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 17 Dec 2013 00:09:13 +0800 Subject: [PATCH 016/105] change oracle jdk 6 to 7 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c2d9b47..aec209e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: java script: "mvn clean install -U" jdk: - - oraclejdk6 - #- oraclejdk7 + #- oraclejdk6 + - oraclejdk7 - openjdk6 - openjdk7 \ No newline at end of file From 37ba18d41f4e814675308ce5e4cc0f0f2f747e3e Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 17 Dec 2013 00:47:53 +0800 Subject: [PATCH 017/105] jetty and others --- .DS_Store | Bin 6148 -> 6148 bytes demo/.DS_Store | Bin 6148 -> 6148 bytes demo/demo.iml | 12 +++++++----- demo/pom.xml | 39 +++++++++++++++++++++++++++++++++++---- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/.DS_Store b/.DS_Store index 344f67ceab9667324694eeaece27fa3dda580ccd..f8b64f14cb1b4abba656ea103bb350b269a79956 100644 GIT binary patch delta 25 gcmZoMXffE(#l&H1YNVrJXlXe)pV?*eH6}e#0Aa}oPXGV_ delta 16 XcmZoMXffE(#WXpc*=O@LCOuIAG?)dv diff --git a/demo/.DS_Store b/demo/.DS_Store index d0ec3957f494e138754c9af45d421508531d9179..8b95c52bd6d4f0f6d3c93a907d6cbeb6ffba0f0e 100644 GIT binary patch delta 110 zcmZoMXfc=|#>B)qu~2NHo+2a5#DLw4m>3yZCi5`f<7HzgVMt^sVn}C5Whj{}##qE* zYHFmTU}$MMc><#gs`TdDjIJ!3**W+*fTnH!$oQRkGQWr=2O|Rm5HT=p4iMSG3;??{ B8B72G delta 69 zcmZoMXfc=|#>B`mu~2NHo+2aD#DLwC4MbQb^E2Jve1+MVW%C2(SB#t4Iruq%iZ)+l Y{?0s^U&NAw0SFiw7??H(h-_g70KftgdH?_b diff --git a/demo/demo.iml b/demo/demo.iml index 999a564..5215d4d 100644 --- a/demo/demo.iml +++ b/demo/demo.iml @@ -20,12 +20,14 @@ - - + + - - - + + + + + diff --git a/demo/pom.xml b/demo/pom.xml index 6a82b09..ba1925e 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -4,22 +4,53 @@ com.github.greengerong demo war - 1.2-SNAPSHOT + 1.0-SNAPSHOT demo Maven Webapp http://maven.apache.org com.github.greengerong prerender-java - 1.1-SNAPSHOT + 1.2-SNAPSHOT demo - org.eclipse.jetty - jetty-maven-plugin + org.mortbay.jetty + maven-jetty-plugin + 6.1.10 + + + + /test + ${project.build.directory}/demo/ + + 10 + foo + 8080 + + + + start-jetty + pre-integration-test + + run + + + 0 + true + + + + stop-jetty + post-integration-test + + stop + + + From 7a50f221225ebf8cbd6e986ca6656057b4c62f3f Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 17 Dec 2013 00:50:37 +0800 Subject: [PATCH 018/105] [maven-release-plugin] prepare release prerender-java-1.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f509c4e..5f9a2c6 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.2-SNAPSHOT + 1.2 jar prerender-java From 693181bddf7ee208945defacce0de58921f9df7e Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 17 Dec 2013 00:50:47 +0800 Subject: [PATCH 019/105] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5f9a2c6..8e1e921 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.2 + 1.3-SNAPSHOT jar prerender-java From ebcacb5e3e6e5c2c722175f28ffc620c93e40420 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 17 Dec 2013 00:54:30 +0800 Subject: [PATCH 020/105] release 1.3 --- README.md | 2 +- demo/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9904e47..8a53402 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.1 + 1.2 ## How it works diff --git a/demo/pom.xml b/demo/pom.xml index ba1925e..d9b1f24 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,7 +11,7 @@ com.github.greengerong prerender-java - 1.2-SNAPSHOT + 1.3-SNAPSHOT From 8f8f5ffd3bd017c3c9f0a342abd5506310def440 Mon Sep 17 00:00:00 2001 From: Joshua Chaitin-Pollak Date: Thu, 23 Jan 2014 11:22:29 -0500 Subject: [PATCH 021/105] Added slf4j dependency and some trace-level logging for the filter to explain why a request is intercepted or not. --- pom.xml | 12 ++--- .../greengerong/PreRenderSEOFilter.java | 45 ++++++++----------- 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/pom.xml b/pom.xml index 8e1e921..518b215 100644 --- a/pom.xml +++ b/pom.xml @@ -36,7 +36,7 @@ 2.4 2.5 4.3.1 - 1.2.9 + 1.7.5 @@ -85,11 +85,11 @@ httpclient ${httpclient.version} - - - - - + + org.slf4j + slf4j-api + ${slf4j.version} + diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 81d3c5f..040af86 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -3,47 +3,35 @@ import static com.google.common.collect.FluentIterable.from; -import java.io.Closeable; -import java.io.IOException; -import java.io.OutputStream; +import java.io.*; import java.net.URI; import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.Enumeration; -import java.util.List; +import java.util.*; import java.util.regex.Pattern; -import javax.servlet.Filter; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; +import javax.servlet.*; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.commons.lang.StringUtils; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHeaders; -import org.apache.http.HttpHost; -import org.apache.http.HttpRequest; -import org.apache.http.HttpResponse; +import org.apache.http.*; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.utils.URIUtils; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.client.*; import org.apache.http.impl.conn.DefaultProxyRoutePlanner; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicHeader; import org.apache.http.message.HeaderGroup; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.base.Predicate; import com.google.common.collect.Lists; public class PreRenderSEOFilter implements Filter { + + private final static Logger log = LoggerFactory.getLogger(PreRenderSEOFilter.class); private FilterConfig filterConfig; @@ -238,42 +226,47 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI final String userAgent = request.getHeader("User-Agent"); final String url = request.getRequestURL().toString(); final String referer = request.getHeader("Referer"); + + log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { - // only respond to GET requests + log.trace("Request is not HTTP GET, not intercepting"); return false; } if (hasEscapedFragment(request)) { - // request has the escape fragment, as defined by google, intercept the request + log.trace("Request Has _escaped_fragment_, intercepting"); return true; } if (StringUtils.isBlank(userAgent)) { - // no User-Agent header, don't intercept + log.trace("Request has blank userAgent, not intercepting"); return false; } if (!isInSearchUserAgent(userAgent)) { - // User-Agent is not a search bot, don't intercept + log.trace("Request User-Agent is not a search bot, don't intercept"); return false; } if (isInResources(url)) { - // request is for a (static) resource, don't intercept + log.trace("request is for a (static) resource, don't intercept"); return false; } final List whiteList = getWhitelist(); if (whiteList != null && !isInWhiteList(url, whiteList)) { + log.trace("Request is whitelisted, intercepting"); return false; } final List blacklist = getBlacklist(); if (blacklist != null && isInBlackList(url, referer, blacklist)) { + log.trace("Request is blacklisted, not intercepting"); return false; } + log.trace("Defaulting to intercept"); return true; } From f429cf7a6d182572ba7fbc280327e384c8b4d490 Mon Sep 17 00:00:00 2001 From: Joshua Chaitin-Pollak Date: Thu, 23 Jan 2014 12:59:16 -0500 Subject: [PATCH 022/105] standardized logging so it would be consistent and searchable by yes/no --- .../java/com/github/greengerong/PreRenderSEOFilter.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 040af86..798d3fc 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -230,7 +230,7 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { - log.trace("Request is not HTTP GET, not intercepting"); + log.trace("Request is not HTTP GET, don't intercept"); return false; } @@ -240,7 +240,7 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI } if (StringUtils.isBlank(userAgent)) { - log.trace("Request has blank userAgent, not intercepting"); + log.trace("Request has blank userAgent, don't intercept"); return false; } @@ -262,11 +262,11 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI final List blacklist = getBlacklist(); if (blacklist != null && isInBlackList(url, referer, blacklist)) { - log.trace("Request is blacklisted, not intercepting"); + log.trace("Request is blacklisted, don't intercept"); return false; } - log.trace("Defaulting to intercept"); + log.trace("Defaulting to request intercepting"); return true; } From 33797d81f93319216698ea5c3517c1eb4cbe7465 Mon Sep 17 00:00:00 2001 From: Joshua Chaitin-Pollak Date: Thu, 23 Jan 2014 13:20:52 -0500 Subject: [PATCH 023/105] standardized logging also fixed a bug: whitelist was actually a blacklist. --- .../github/greengerong/PreRenderSEOFilter.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 798d3fc..aa117a2 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -230,43 +230,43 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { - log.trace("Request is not HTTP GET, don't intercept"); + log.trace("Request is not HTTP GET; intercept: no"); return false; } if (hasEscapedFragment(request)) { - log.trace("Request Has _escaped_fragment_, intercepting"); + log.trace("Request Has _escaped_fragment_; intercept: yes"); return true; } if (StringUtils.isBlank(userAgent)) { - log.trace("Request has blank userAgent, don't intercept"); + log.trace("Request has blank userAgent; intercept: no"); return false; } if (!isInSearchUserAgent(userAgent)) { - log.trace("Request User-Agent is not a search bot, don't intercept"); + log.trace("Request User-Agent is not a search bot; intercept: no"); return false; } if (isInResources(url)) { - log.trace("request is for a (static) resource, don't intercept"); + log.trace("request is for a (static) resource; intercept: no"); return false; } final List whiteList = getWhitelist(); if (whiteList != null && !isInWhiteList(url, whiteList)) { - log.trace("Request is whitelisted, intercepting"); - return false; + log.trace("Request is whitelisted; intercept: yes"); + return true; } final List blacklist = getBlacklist(); if (blacklist != null && isInBlackList(url, referer, blacklist)) { - log.trace("Request is blacklisted, don't intercept"); + log.trace("Request is blacklisted; intercept: no"); return false; } - log.trace("Defaulting to request intercepting"); + log.trace("Defaulting to request intercept: yes"); return true; } From 39f4b67f1cb0bd9367a7e281134de5af5421a10a Mon Sep 17 00:00:00 2001 From: Joshua Chaitin-Pollak Date: Thu, 23 Jan 2014 14:29:36 -0500 Subject: [PATCH 024/105] fixed the semantics and logging for whitelists and blacklists --- .../java/com/github/greengerong/PreRenderSEOFilter.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index aa117a2..5593743 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -256,13 +256,13 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI final List whiteList = getWhitelist(); if (whiteList != null && !isInWhiteList(url, whiteList)) { - log.trace("Request is whitelisted; intercept: yes"); - return true; + log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); + return false; } final List blacklist = getBlacklist(); if (blacklist != null && isInBlackList(url, referer, blacklist)) { - log.trace("Request is blacklisted; intercept: no"); + log.trace("Blacklist is enabled, and this request is listed; intercept: no"); return false; } From c243ec94a3f6efbd4c21ac2f51508d05a6831095 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Sun, 9 Feb 2014 23:16:21 +0800 Subject: [PATCH 025/105] [maven-release-plugin] prepare release prerender-java-1.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 518b215..c6e4608 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.3-SNAPSHOT + 1.3 jar prerender-java From 3fd81155cd8686054e729cc35cf8dfbc2f8ae1b7 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Sun, 9 Feb 2014 23:16:42 +0800 Subject: [PATCH 026/105] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c6e4608..67fc882 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.3 + 1.4-SNAPSHOT jar prerender-java From ec79f67e606a101f678dc72e8c59ecbaf2b501ec Mon Sep 17 00:00:00 2001 From: greengerongg Date: Sun, 9 Feb 2014 23:21:27 +0800 Subject: [PATCH 027/105] release for 1.3 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8a53402..3d70377 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.2 + 1.3 ## How it works From 76d8e36e404cc24125cf4ccf0e7c08cbb7027e4d Mon Sep 17 00:00:00 2001 From: greengerongg Date: Mon, 10 Feb 2014 10:26:21 +0800 Subject: [PATCH 028/105] change demo version --- demo/pom.xml | 2 +- prerender-java.iml | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/demo/pom.xml b/demo/pom.xml index d9b1f24..67a42c8 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,7 +11,7 @@ com.github.greengerong prerender-java - 1.3-SNAPSHOT + 1.3 diff --git a/prerender-java.iml b/prerender-java.iml index 2dc2562..f713073 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -12,13 +12,16 @@ - + + - - - + + + + + From 80073933543a2655c3f3a8aa2f81055d9bbe327d Mon Sep 17 00:00:00 2001 From: greengerongg Date: Fri, 14 Feb 2014 20:54:17 +0800 Subject: [PATCH 029/105] add prerender token for public prerender service --- demo/src/main/webapp/index.jsp | 1 + .../greengerong/PreRenderSEOFilter.java | 158 ++++++++++-------- 2 files changed, 91 insertions(+), 68 deletions(-) diff --git a/demo/src/main/webapp/index.jsp b/demo/src/main/webapp/index.jsp index 63d1612..4a09353 100644 --- a/demo/src/main/webapp/index.jsp +++ b/demo/src/main/webapp/index.jsp @@ -1,5 +1,6 @@ + diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 5593743..6f143c0 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,24 +1,16 @@ package com.github.greengerong; -import static com.google.common.collect.FluentIterable.from; - -import java.io.*; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.*; -import java.util.regex.Pattern; - -import javax.servlet.*; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - +import com.google.common.base.Predicate; +import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.apache.http.*; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.utils.URIUtils; -import org.apache.http.impl.client.*; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.DefaultProxyRoutePlanner; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicHeader; @@ -26,30 +18,42 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Predicate; -import com.google.common.collect.Lists; +import javax.servlet.*; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.Closeable; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.List; +import java.util.regex.Pattern; + +import static com.google.common.collect.FluentIterable.from; public class PreRenderSEOFilter implements Filter { - + private final static Logger log = LoggerFactory.getLogger(PreRenderSEOFilter.class); private FilterConfig filterConfig; - + private CloseableHttpClient httpClient; @Override public void init(FilterConfig filterConfig) throws ServletException { this.filterConfig = filterConfig; - + HttpClientBuilder builder = HttpClients.custom(); - + final String proxy = filterConfig.getInitParameter("proxy"); if (StringUtils.isNotBlank(proxy)) { final int proxyPort = Integer.parseInt(filterConfig.getInitParameter("proxyPort")); - DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); + DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); builder = builder.setRoutePlanner(routePlanner); } - + builder = builder.setConnectionManager(new PoolingHttpClientConnectionManager()); httpClient = builder.build(); } @@ -61,9 +65,9 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo final HttpServletRequest request = (HttpServletRequest) servletRequest; final HttpServletResponse response = (HttpServletResponse) servletResponse; if (shouldShowPrerenderedPage(request)) { - if (proxyPrerenderedPageResponse(request, response)) { - return; - } + if (proxyPrerenderedPageResponse(request, response)) { + return; + } } } catch (Exception e) { } @@ -74,6 +78,7 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer final String apiUrl = getApiUrl(getFullUrl(request)); final HttpGet getMethod = new HttpGet(apiUrl); copyRequestHeaders(request, getMethod); + withPrerenderToken(getMethod); CloseableHttpResponse httpResponse = httpClient.execute(getMethod); try { if (httpResponse.getStatusLine().getStatusCode() == 200) { @@ -87,7 +92,17 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer return false; } - /** Copy proxied response headers back to the servlet client. */ + private void withPrerenderToken(HttpRequest proxyRequest) { + final String token = filterConfig.getInitParameter("prerenderToken"); + //for new version prerender with token. + if (StringUtils.isNotBlank(token)) { + proxyRequest.addHeader("X-Prerender-Token", token); + } + } + + /** + * Copy proxied response headers back to the servlet client. + */ protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { for (Header header : proxyResponse.getAllHeaders()) { if (hopByHopHeaders.containsHeader(header.getName())) @@ -95,8 +110,10 @@ protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletRespon servletResponse.addHeader(header.getName(), header.getValue()); } } - - /** Copy response body data (the entity) from the proxy to the servlet client. */ + + /** + * Copy response body data (the entity) from the proxy to the servlet client. + */ protected void copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) throws IOException { HttpEntity entity = proxyResponse.getEntity(); if (entity != null) { @@ -108,60 +125,65 @@ protected void copyResponseEntity(HttpResponse proxyResponse, HttpServletRespons } } } - + protected void closeQuietly(Closeable closeable) { try { closeable.close(); } catch (IOException e) { } } - - /** These are the "hop-by-hop" headers that should not be copied. + + /** + * These are the "hop-by-hop" headers that should not be copied. * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html * I use an HttpClient HeaderGroup class instead of Set because this * approach does case insensitive lookup faster. */ protected static final HeaderGroup hopByHopHeaders; + static { - hopByHopHeaders = new HeaderGroup(); - String[] headers = new String[] { - "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", - "TE", "Trailers", "Transfer-Encoding", "Upgrade" }; - for (String header : headers) { - hopByHopHeaders.addHeader(new BasicHeader(header, null)); - } + hopByHopHeaders = new HeaderGroup(); + String[] headers = new String[]{ + "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", + "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; + for (String header : headers) { + hopByHopHeaders.addHeader(new BasicHeader(header, null)); + } } - - /** Copy request headers from the servlet client to the proxy request. - * @throws URISyntaxException */ + + /** + * Copy request headers from the servlet client to the proxy request. + * + * @throws URISyntaxException + */ protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) throws URISyntaxException { - // Get an Enumeration of all of the header names sent by the client - Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); - while (enumerationOfHeaderNames.hasMoreElements()) { - String headerName = (String) enumerationOfHeaderNames.nextElement(); - //Instead the content-length is effectively set via InputStreamEntity - if (headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH)) - continue; - if (hopByHopHeaders.containsHeader(headerName)) - continue; - - Enumeration headers = servletRequest.getHeaders(headerName); - while (headers.hasMoreElements()) {//sometimes more than one value - String headerValue = (String) headers.nextElement(); - // In case the proxy host is running multiple virtual servers, - // rewrite the Host header to ensure that we get content from - // the correct virtual server - if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { - HttpHost host = URIUtils.extractHost(new URI(getPrerenderServiceUrl())); - headerValue = host.getHostName(); - if (host.getPort() != -1) - headerValue += ":"+host.getPort(); - } - proxyRequest.addHeader(headerName, headerValue); + // Get an Enumeration of all of the header names sent by the client + Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); + while (enumerationOfHeaderNames.hasMoreElements()) { + String headerName = (String) enumerationOfHeaderNames.nextElement(); + //Instead the content-length is effectively set via InputStreamEntity + if (headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH)) + continue; + if (hopByHopHeaders.containsHeader(headerName)) + continue; + + Enumeration headers = servletRequest.getHeaders(headerName); + while (headers.hasMoreElements()) {//sometimes more than one value + String headerValue = (String) headers.nextElement(); + // In case the proxy host is running multiple virtual servers, + // rewrite the Host header to ensure that we get content from + // the correct virtual server + if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { + HttpHost host = URIUtils.extractHost(new URI(getPrerenderServiceUrl())); + headerValue = host.getHostName(); + if (host.getPort() != -1) + headerValue += ":" + host.getPort(); + } + proxyRequest.addHeader(headerName, headerValue); + } } - } } - + private String getFullUrl(HttpServletRequest request) { final StringBuffer url = request.getRequestURL(); final String queryString = request.getQueryString(); @@ -226,14 +248,14 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI final String userAgent = request.getHeader("User-Agent"); final String url = request.getRequestURL().toString(); final String referer = request.getHeader("Referer"); - + log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { log.trace("Request is not HTTP GET; intercept: no"); return false; } - + if (hasEscapedFragment(request)) { log.trace("Request Has _escaped_fragment_; intercept: yes"); return true; @@ -284,7 +306,7 @@ private String getApiUrl(String url) { private String getPrerenderServiceUrl() { final String prerenderServiceUrl = filterConfig.getInitParameter("prerenderServiceUrl"); - return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : "http://prerender.herokuapp.com/"; + return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : "'http://service.prerender.io/'"; } private boolean isInBlackList(final String url, final String referer, List blacklist) { From 5593ce2ebdbf25e9044b810f74d75384f5ea1209 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Fri, 14 Feb 2014 20:57:07 +0800 Subject: [PATCH 030/105] read me for prerenderToken --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 3d70377..482df48 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,16 @@ As an alternative, you can pass `prerender_service_url` in the options object du config filter init param with "prerenderServiceUrl"; ``` +### prerender service token + +If you want to use token with the prerender service, you can config it. + + +``` xml + config filter init param with "prerenderToken"; +``` + + ## Testing If you want to make sure your pages are rendering correctly: From 2d52b156cff6c79261fd3541b9aece5c7d8a293e Mon Sep 17 00:00:00 2001 From: greengerongg Date: Fri, 14 Feb 2014 22:03:31 +0800 Subject: [PATCH 031/105] refactor --- .../greengerong/PreRenderSEOFilter.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 6f143c0..15f03fb 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -70,6 +70,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo } } } catch (Exception e) { + log.error("Prerender service error", e); } filterChain.doFilter(servletRequest, servletResponse); } @@ -105,9 +106,9 @@ private void withPrerenderToken(HttpRequest proxyRequest) { */ protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { for (Header header : proxyResponse.getAllHeaders()) { - if (hopByHopHeaders.containsHeader(header.getName())) - continue; - servletResponse.addHeader(header.getName(), header.getValue()); + if (!hopByHopHeaders.containsHeader(header.getName())) { + servletResponse.addHeader(header.getName(), header.getValue()); + } } } @@ -130,6 +131,7 @@ protected void closeQuietly(Closeable closeable) { try { closeable.close(); } catch (IOException e) { + log.error("Close proxy error", e); } } @@ -162,24 +164,22 @@ protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest while (enumerationOfHeaderNames.hasMoreElements()) { String headerName = (String) enumerationOfHeaderNames.nextElement(); //Instead the content-length is effectively set via InputStreamEntity - if (headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH)) - continue; - if (hopByHopHeaders.containsHeader(headerName)) - continue; - - Enumeration headers = servletRequest.getHeaders(headerName); - while (headers.hasMoreElements()) {//sometimes more than one value - String headerValue = (String) headers.nextElement(); - // In case the proxy host is running multiple virtual servers, - // rewrite the Host header to ensure that we get content from - // the correct virtual server - if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { - HttpHost host = URIUtils.extractHost(new URI(getPrerenderServiceUrl())); - headerValue = host.getHostName(); - if (host.getPort() != -1) - headerValue += ":" + host.getPort(); + if (!headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { + Enumeration headers = servletRequest.getHeaders(headerName); + while (headers.hasMoreElements()) {//sometimes more than one value + String headerValue = (String) headers.nextElement(); + // In case the proxy host is running multiple virtual servers, + // rewrite the Host header to ensure that we get content from + // the correct virtual server + if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { + HttpHost host = URIUtils.extractHost(new URI(getPrerenderServiceUrl())); + headerValue = host.getHostName(); + if (host.getPort() != -1) { + headerValue += ":" + host.getPort(); + } + } + proxyRequest.addHeader(headerName, headerValue); } - proxyRequest.addHeader(headerName, headerValue); } } } From c877ccd91808843f75aac8f908af1153d0c90c85 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Fri, 14 Feb 2014 23:28:11 +0800 Subject: [PATCH 032/105] [maven-release-plugin] prepare release prerender-java-1.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 67fc882..3b88c16 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.4-SNAPSHOT + 1.4 jar prerender-java From 420e4725798005ef83f9f578ce6bb5a4b5adea1e Mon Sep 17 00:00:00 2001 From: greengerongg Date: Fri, 14 Feb 2014 23:28:28 +0800 Subject: [PATCH 033/105] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3b88c16..db6c0ff 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.4 + 1.5-SNAPSHOT jar prerender-java From 3a42bf4cd931ada401729e6f02663ab85f09ca87 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Fri, 14 Feb 2014 23:34:52 +0800 Subject: [PATCH 034/105] [maven-release-plugin] prepare for next development iteration --- README.md | 2 +- demo/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 482df48..0b071ff 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.3 + 1.4 ## How it works diff --git a/demo/pom.xml b/demo/pom.xml index 67a42c8..45f5542 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,7 +11,7 @@ com.github.greengerong prerender-java - 1.3 + 1.4 From 889461e30f7d65a22ca4429f8f457b884aa05107 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Sat, 15 Feb 2014 00:30:23 +0800 Subject: [PATCH 035/105] [maven-release-plugin] prepare for next development iteration --- README.md | 10 ++++ .../greengerong/PreRenderEventHandler.java | 13 +++++ .../greengerong/PreRenderSEOFilter.java | 54 ++++++++++++++++--- 3 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/github/greengerong/PreRenderEventHandler.java diff --git a/README.md b/README.md index 0b071ff..b9a7d5f 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,16 @@ If you want to use token with the prerender service, you can config it. ``` +### prerender event handler + +If you want to cache the caching, analytics, log or others, you can config it. It should be instance of "com.github.greengerong.PreRenderEventHandler" + + +``` xml + config filter init param with "preRenderEventHandler"; +``` + + ## Testing If you want to make sure your pages are rendering correctly: diff --git a/src/main/java/com/github/greengerong/PreRenderEventHandler.java b/src/main/java/com/github/greengerong/PreRenderEventHandler.java new file mode 100644 index 0000000..9e0d74e --- /dev/null +++ b/src/main/java/com/github/greengerong/PreRenderEventHandler.java @@ -0,0 +1,13 @@ +package com.github.greengerong; + + +import org.apache.http.HttpResponse; + +import javax.servlet.http.HttpServletRequest; + +public interface PreRenderEventHandler { + + String beforeRender(HttpServletRequest clientRequest); + + void afterRender(HttpServletRequest clientRequest, HttpResponse prerenderResponse); +} diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 15f03fb..30fbcfc 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -24,6 +24,7 @@ import java.io.Closeable; import java.io.IOException; import java.io.OutputStream; +import java.io.PrintWriter; import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; @@ -40,11 +41,28 @@ public class PreRenderSEOFilter implements Filter { private FilterConfig filterConfig; private CloseableHttpClient httpClient; + private PreRenderEventHandler preRenderEventHandler; @Override public void init(FilterConfig filterConfig) throws ServletException { this.filterConfig = filterConfig; + this.httpClient = getHttpClient(filterConfig); + this.preRenderEventHandler = getEventHandler(filterConfig); + } + private PreRenderEventHandler getEventHandler(FilterConfig filterConfig) { + final String preRenderEventHandler = filterConfig.getInitParameter("preRenderEventHandler"); + if (StringUtils.isNotBlank(preRenderEventHandler)) { + try { + return (PreRenderEventHandler) Class.forName(preRenderEventHandler).newInstance(); + } catch (Exception e) { + log.error("PreRenderEventHandler class not find or can not new a instance", e); + } + } + return null; + } + + private CloseableHttpClient getHttpClient(FilterConfig filterConfig) { HttpClientBuilder builder = HttpClients.custom(); final String proxy = filterConfig.getInitParameter("proxy"); @@ -55,7 +73,7 @@ public void init(FilterConfig filterConfig) throws ServletException { } builder = builder.setConnectionManager(new PoolingHttpClientConnectionManager()); - httpClient = builder.build(); + return builder.build(); } @Override @@ -65,7 +83,8 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo final HttpServletRequest request = (HttpServletRequest) servletRequest; final HttpServletResponse response = (HttpServletResponse) servletResponse; if (shouldShowPrerenderedPage(request)) { - if (proxyPrerenderedPageResponse(request, response)) { + + if (beforeRender(request, response) || proxyPrerenderedPageResponse(request, response)) { return; } } @@ -75,24 +94,45 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo filterChain.doFilter(servletRequest, servletResponse); } + private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) throws IOException { + if (preRenderEventHandler != null) { + final String html = preRenderEventHandler.beforeRender(request); + if (StringUtils.isNotBlank(html)) { + final PrintWriter writer = response.getWriter(); + writer.write(html); + writer.flush(); + closeQuietly(writer); + return true; + } + } + return false; + } + private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException { final String apiUrl = getApiUrl(getFullUrl(request)); final HttpGet getMethod = new HttpGet(apiUrl); copyRequestHeaders(request, getMethod); withPrerenderToken(getMethod); - CloseableHttpResponse httpResponse = httpClient.execute(getMethod); + CloseableHttpResponse proxyResponse = httpClient.execute(getMethod); try { - if (httpResponse.getStatusLine().getStatusCode() == 200) { - copyResponseHeaders(httpResponse, response); - copyResponseEntity(httpResponse, response); + if (proxyResponse.getStatusLine().getStatusCode() == 200) { + afterRender(request, proxyResponse); + copyResponseHeaders(proxyResponse, response); + copyResponseEntity(proxyResponse, response); return true; } } finally { - httpResponse.close(); + closeQuietly(proxyResponse); } return false; } + private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse) { + if (preRenderEventHandler != null) { + preRenderEventHandler.afterRender(request, proxyResponse); + } + } + private void withPrerenderToken(HttpRequest proxyRequest) { final String token = filterConfig.getInitParameter("prerenderToken"); //for new version prerender with token. From 97e3d4f69ec647567d389d009342bb1150b04ba2 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Sat, 15 Feb 2014 00:36:15 +0800 Subject: [PATCH 036/105] add some crawler --- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 30fbcfc..b332a68 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -246,7 +246,7 @@ public void destroy() { private List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", - "facebookexternalhit", "twitterbot"); + "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly"); final String crawlerUserAgentsFromConfig = filterConfig.getInitParameter("crawlerUserAgents"); if (StringUtils.isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); From 77fe1149d30cbf0f49effd512e238a37a9a96009 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Sat, 15 Feb 2014 17:26:11 +0800 Subject: [PATCH 037/105] refactor --- .../greengerong/PreRenderSEOFilter.java | 116 +++--------------- .../github/greengerong/PrerenderConfig.java | 100 +++++++++++++++ 2 files changed, 118 insertions(+), 98 deletions(-) create mode 100644 src/main/java/com/github/greengerong/PrerenderConfig.java diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index b332a68..1a83088 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -2,17 +2,12 @@ import com.google.common.base.Predicate; -import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.apache.http.*; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.utils.URIUtils; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.DefaultProxyRoutePlanner; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicHeader; import org.apache.http.message.HeaderGroup; import org.slf4j.Logger; @@ -27,7 +22,6 @@ import java.io.PrintWriter; import java.net.URI; import java.net.URISyntaxException; -import java.util.Arrays; import java.util.Enumeration; import java.util.List; import java.util.regex.Pattern; @@ -37,43 +31,16 @@ public class PreRenderSEOFilter implements Filter { private final static Logger log = LoggerFactory.getLogger(PreRenderSEOFilter.class); - - private FilterConfig filterConfig; - private CloseableHttpClient httpClient; private PreRenderEventHandler preRenderEventHandler; + private PrerenderConfig prerenderConfig; + public static final int HTTP_OK = 200; @Override public void init(FilterConfig filterConfig) throws ServletException { - this.filterConfig = filterConfig; - this.httpClient = getHttpClient(filterConfig); - this.preRenderEventHandler = getEventHandler(filterConfig); - } - - private PreRenderEventHandler getEventHandler(FilterConfig filterConfig) { - final String preRenderEventHandler = filterConfig.getInitParameter("preRenderEventHandler"); - if (StringUtils.isNotBlank(preRenderEventHandler)) { - try { - return (PreRenderEventHandler) Class.forName(preRenderEventHandler).newInstance(); - } catch (Exception e) { - log.error("PreRenderEventHandler class not find or can not new a instance", e); - } - } - return null; - } - - private CloseableHttpClient getHttpClient(FilterConfig filterConfig) { - HttpClientBuilder builder = HttpClients.custom(); - - final String proxy = filterConfig.getInitParameter("proxy"); - if (StringUtils.isNotBlank(proxy)) { - final int proxyPort = Integer.parseInt(filterConfig.getInitParameter("proxyPort")); - DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); - builder = builder.setRoutePlanner(routePlanner); - } - - builder = builder.setConnectionManager(new PoolingHttpClientConnectionManager()); - return builder.build(); + this.prerenderConfig = new PrerenderConfig(filterConfig); + this.httpClient = prerenderConfig.getHttpClient(); + this.preRenderEventHandler = prerenderConfig.getEventHandler(); } @Override @@ -115,7 +82,7 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer withPrerenderToken(getMethod); CloseableHttpResponse proxyResponse = httpClient.execute(getMethod); try { - if (proxyResponse.getStatusLine().getStatusCode() == 200) { + if (proxyResponse.getStatusLine().getStatusCode() == HTTP_OK) { afterRender(request, proxyResponse); copyResponseHeaders(proxyResponse, response); copyResponseEntity(proxyResponse, response); @@ -134,7 +101,7 @@ private void afterRender(HttpServletRequest request, CloseableHttpResponse proxy } private void withPrerenderToken(HttpRequest proxyRequest) { - final String token = filterConfig.getInitParameter("prerenderToken"); + final String token = prerenderConfig.getPrerenderToken(); //for new version prerender with token. if (StringUtils.isNotBlank(token)) { proxyRequest.addHeader("X-Prerender-Token", token); @@ -169,7 +136,9 @@ protected void copyResponseEntity(HttpResponse proxyResponse, HttpServletRespons protected void closeQuietly(Closeable closeable) { try { - closeable.close(); + if (closeable != null) { + closeable.close(); + } } catch (IOException e) { log.error("Close proxy error", e); } @@ -212,7 +181,7 @@ protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest // rewrite the Host header to ensure that we get content from // the correct virtual server if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { - HttpHost host = URIUtils.extractHost(new URI(getPrerenderServiceUrl())); + HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); headerValue = host.getHostName(); if (host.getPort() != -1) { headerValue += ":" + host.getPort(); @@ -236,52 +205,8 @@ private String getFullUrl(HttpServletRequest request) { @Override public void destroy() { - filterConfig = null; - try { - httpClient.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - private List getCrawlerUserAgents() { - List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", - "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly"); - final String crawlerUserAgentsFromConfig = filterConfig.getInitParameter("crawlerUserAgents"); - if (StringUtils.isNotBlank(crawlerUserAgentsFromConfig)) { - crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); - } - - return crawlerUserAgents; - } - - private List getExtensionsToIgnore() { - List extensionsToIgnore = Lists.newArrayList(".js", ".css", ".less", ".png", ".jpg", ".jpeg", - ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", - ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", - ".iso", ".flv", ".m4v", ".torrent"); - final String extensionsToIgnoreFromConfig = filterConfig.getInitParameter("extensionsToIgnore"); - if (StringUtils.isNotBlank(extensionsToIgnoreFromConfig)) { - extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); - } - - return extensionsToIgnore; - } - - private List getWhitelist() { - final String whitelist = filterConfig.getInitParameter("whitelist"); - if (StringUtils.isNotBlank(whitelist)) { - return Arrays.asList(whitelist.trim().split(",")); - } - return null; - } - - private List getBlacklist() { - final String blacklist = filterConfig.getInitParameter("blacklist"); - if (StringUtils.isNotBlank(blacklist)) { - return Arrays.asList(blacklist.trim().split(",")); - } - return null; + prerenderConfig = null; + closeQuietly(httpClient); } private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { @@ -316,13 +241,13 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI return false; } - final List whiteList = getWhitelist(); + final List whiteList = prerenderConfig.getWhitelist(); if (whiteList != null && !isInWhiteList(url, whiteList)) { log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); return false; } - final List blacklist = getBlacklist(); + final List blacklist = prerenderConfig.getBlacklist(); if (blacklist != null && isInBlackList(url, referer, blacklist)) { log.trace("Blacklist is enabled, and this request is listed; intercept: no"); return false; @@ -337,18 +262,13 @@ private boolean hasEscapedFragment(HttpServletRequest request) { } private String getApiUrl(String url) { - String prerenderServiceUrl = getPrerenderServiceUrl(); + String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); if (!prerenderServiceUrl.endsWith("/")) { prerenderServiceUrl += "/"; } return prerenderServiceUrl + url; } - private String getPrerenderServiceUrl() { - final String prerenderServiceUrl = filterConfig.getInitParameter("prerenderServiceUrl"); - return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : "'http://service.prerender.io/'"; - } - private boolean isInBlackList(final String url, final String referer, List blacklist) { return from(blacklist).anyMatch(new Predicate() { @Override @@ -370,7 +290,7 @@ public boolean apply(String regex) { } private boolean isInResources(final String url) { - return from(getExtensionsToIgnore()).anyMatch(new Predicate() { + return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { @Override public boolean apply(String item) { return url.contains(item.toLowerCase()); @@ -379,7 +299,7 @@ public boolean apply(String item) { } private boolean isInSearchUserAgent(final String userAgent) { - return from(getCrawlerUserAgents()).anyMatch(new Predicate() { + return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { @Override public boolean apply(String item) { return userAgent.toLowerCase().indexOf(item.toLowerCase()) >= 0; diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java new file mode 100644 index 0000000..6653666 --- /dev/null +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -0,0 +1,100 @@ +package com.github.greengerong; + + +import com.google.common.collect.Lists; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpHost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.DefaultProxyRoutePlanner; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.FilterConfig; +import java.util.Arrays; +import java.util.List; + +public class PrerenderConfig { + private final static Logger log = LoggerFactory.getLogger(PrerenderConfig.class); + private FilterConfig filterConfig; + + public PrerenderConfig(FilterConfig filterConfig) { + this.filterConfig = filterConfig; + } + + public PreRenderEventHandler getEventHandler() { + final String preRenderEventHandler = filterConfig.getInitParameter("preRenderEventHandler"); + if (StringUtils.isNotBlank(preRenderEventHandler)) { + try { + return (PreRenderEventHandler) Class.forName(preRenderEventHandler).newInstance(); + } catch (Exception e) { + log.error("PreRenderEventHandler class not find or can not new a instance", e); + } + } + return null; + } + + public CloseableHttpClient getHttpClient() { + HttpClientBuilder builder = HttpClients.custom(); + final String proxy = filterConfig.getInitParameter("proxy"); + if (StringUtils.isNotBlank(proxy)) { + final int proxyPort = Integer.parseInt(filterConfig.getInitParameter("proxyPort")); + DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); + builder = builder.setRoutePlanner(routePlanner); + } + + builder = builder.setConnectionManager(new PoolingHttpClientConnectionManager()); + return builder.build(); + } + + public String getPrerenderToken() { + return filterConfig.getInitParameter("prerenderToken"); + } + + public List getCrawlerUserAgents() { + List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", + "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly"); + final String crawlerUserAgentsFromConfig = filterConfig.getInitParameter("crawlerUserAgents"); + if (StringUtils.isNotBlank(crawlerUserAgentsFromConfig)) { + crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); + } + + return crawlerUserAgents; + } + + public List getExtensionsToIgnore() { + List extensionsToIgnore = Lists.newArrayList(".js", ".css", ".less", ".png", ".jpg", ".jpeg", + ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", + ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", + ".iso", ".flv", ".m4v", ".torrent"); + final String extensionsToIgnoreFromConfig = filterConfig.getInitParameter("extensionsToIgnore"); + if (StringUtils.isNotBlank(extensionsToIgnoreFromConfig)) { + extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); + } + + return extensionsToIgnore; + } + + public List getWhitelist() { + final String whitelist = filterConfig.getInitParameter("whitelist"); + if (StringUtils.isNotBlank(whitelist)) { + return Arrays.asList(whitelist.trim().split(",")); + } + return null; + } + + public List getBlacklist() { + final String blacklist = filterConfig.getInitParameter("blacklist"); + if (StringUtils.isNotBlank(blacklist)) { + return Arrays.asList(blacklist.trim().split(",")); + } + return null; + } + + public String getPrerenderServiceUrl() { + final String prerenderServiceUrl = filterConfig.getInitParameter("prerenderServiceUrl"); + return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : "'http://service.prerender.io/'"; + } +} From 6ff5481e7d999fc7c638e21a05ea86363010934f Mon Sep 17 00:00:00 2001 From: greengerongg Date: Sat, 15 Feb 2014 18:43:43 +0800 Subject: [PATCH 038/105] all test for ci --- pom.xml | 6 - prerender-java.iml | 2 +- .../greengerong/PreRenderSEOFilter.java | 12 +- .../greengerong/PreRenderSEOFilterTest.java | 265 ++++++++++++++++++ 4 files changed, 276 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index db6c0ff..af40fe4 100644 --- a/pom.xml +++ b/pom.xml @@ -40,12 +40,6 @@ - - junit - junit - 4.11 - test - com.google.guava guava diff --git a/prerender-java.iml b/prerender-java.iml index f713073..cbd3c69 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -10,9 +10,9 @@ + - diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 1a83088..69976cf 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -39,10 +39,14 @@ public class PreRenderSEOFilter implements Filter { @Override public void init(FilterConfig filterConfig) throws ServletException { this.prerenderConfig = new PrerenderConfig(filterConfig); - this.httpClient = prerenderConfig.getHttpClient(); + this.httpClient = getHttpClient(); this.preRenderEventHandler = prerenderConfig.getEventHandler(); } + protected CloseableHttpClient getHttpClient() { + return prerenderConfig.getHttpClient(); + } + @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { @@ -77,7 +81,7 @@ private boolean beforeRender(HttpServletRequest request, HttpServletResponse res private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException { final String apiUrl = getApiUrl(getFullUrl(request)); - final HttpGet getMethod = new HttpGet(apiUrl); + final HttpGet getMethod = getHttpGet(apiUrl); copyRequestHeaders(request, getMethod); withPrerenderToken(getMethod); CloseableHttpResponse proxyResponse = httpClient.execute(getMethod); @@ -94,6 +98,10 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer return false; } + protected HttpGet getHttpGet(String apiUrl) { + return new HttpGet(apiUrl); + } + private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse) { if (preRenderEventHandler != null) { preRenderEventHandler.afterRender(request, proxyResponse); diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 20beb9b..1ac7521 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -1,4 +1,269 @@ package com.github.greengerong; +import com.google.common.collect.Maps; +import org.apache.http.Header; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.Enumeration; +import java.util.HashMap; + +import static org.mockito.Mockito.*; + +@RunWith(MockitoJUnitRunner.class) public class PreRenderSEOFilterTest { + + public static final int NOT_FOUND = 404; + private PreRenderSEOFilter preRenderSEOFilter; + + @Mock + private CloseableHttpClient httpClient; + + @Mock + private FilterConfig filterConfig; + + @Mock + private HttpServletRequest servletRequest; + + @Mock + private HttpServletResponse servletResponse; + + @Mock + private FilterChain filterChain; + + @Mock + private HttpGet httpGet; + + @Before + public void setUp() throws Exception { + preRenderSEOFilter = new PreRenderSEOFilter() { + @Override + protected CloseableHttpClient getHttpClient() { + return httpClient; + } + + @Override + protected HttpGet getHttpGet(String apiUrl) { + return httpGet; + } + }; + } + + @Test + public void should_not_handle_when_non_get_request() throws Exception { + //given + preRenderSEOFilter.init(filterConfig); + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer()); + when(servletRequest.getMethod()).thenReturn(HttpPost.METHOD_NAME); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_handle_when_url_with_escaped_fragment_() throws Exception { + //given + preRenderSEOFilter.init(filterConfig); + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put("_escaped_fragment_", ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_user_agent_is_not_crawler() throws Exception { + //given + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("no"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_url_is_a_resource() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test.js")); + when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_white_list_is_not_empty_and_url_is_not_in_white_list() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + when(filterConfig.getInitParameter("whitelist")).thenReturn("whitelist1,whitelist2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_black_list_is_not_empty_and_url_is_in_black_list() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + when(filterConfig.getInitParameter("blacklist")).thenReturn("blacklist1,http://localhost/test"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and_white_list_is_empty_and_black_list_is_empty() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put("_escaped_fragment_", ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_every_thing_is_ok_but_prerender_server_response_is_not_200() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put("_escaped_fragment_", ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(NOT_FOUND); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + + @Test + public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and_in_white_list_and_not_in_black_list() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + when(filterConfig.getInitParameter("whitelist")).thenReturn("whitelist1,http://localhost/test"); + when(filterConfig.getInitParameter("blacklist")).thenReturn("blacklist1,blacklist2"); + + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put("_escaped_fragment_", ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } } From b01ff4bce2e17a256486c906a1b4d4e3be902df9 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Mon, 17 Feb 2014 18:25:28 +0800 Subject: [PATCH 039/105] add log for proxy url --- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 69976cf..9b1e296 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -81,11 +81,14 @@ private boolean beforeRender(HttpServletRequest request, HttpServletResponse res private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException { final String apiUrl = getApiUrl(getFullUrl(request)); + log.trace(String.format("Prerender proxy will send request to:%s", apiUrl)); final HttpGet getMethod = getHttpGet(apiUrl); copyRequestHeaders(request, getMethod); withPrerenderToken(getMethod); - CloseableHttpResponse proxyResponse = httpClient.execute(getMethod); + CloseableHttpResponse proxyResponse = null; + try { + proxyResponse = httpClient.execute(getMethod); if (proxyResponse.getStatusLine().getStatusCode() == HTTP_OK) { afterRender(request, proxyResponse); copyResponseHeaders(proxyResponse, response); From e1e2272b6b2a5e8a563059de19978619d91847a3 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Mon, 17 Feb 2014 21:25:01 +0800 Subject: [PATCH 040/105] [maven-release-plugin] prepare release prerender-java-1.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index af40fe4..7a79e27 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.5-SNAPSHOT + 1.5 jar prerender-java From 7a2e75fa3685f219c7169dfe16f1c6f8d95964e8 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Mon, 17 Feb 2014 21:25:17 +0800 Subject: [PATCH 041/105] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7a79e27..af40fe4 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.5 + 1.5-SNAPSHOT jar prerender-java From c5c56c4b0665c8d2e004399e9d782e2643d7e8e3 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 07:21:44 +0800 Subject: [PATCH 042/105] =?UTF-8?q?remove=20=E2=80=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/github/greengerong/PrerenderConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 6653666..c5daa05 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -95,6 +95,6 @@ public List getBlacklist() { public String getPrerenderServiceUrl() { final String prerenderServiceUrl = filterConfig.getInitParameter("prerenderServiceUrl"); - return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : "'http://service.prerender.io/'"; + return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : "http://service.prerender.io/"; } } From 5255fccaa0428e91735ed440803132337272d4cc Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 07:24:35 +0800 Subject: [PATCH 043/105] [maven-release-plugin] prepare release prerender-java-1.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index af40fe4..7a79e27 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.5-SNAPSHOT + 1.5 jar prerender-java From 05a617a3f1d99103ff3a5f99a7bb6d5d73331f1b Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 07:27:52 +0800 Subject: [PATCH 044/105] prepare release prerender-java-1.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7a79e27..eee3653 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.5 + 1.6-SNAPSHOT jar prerender-java From d77cd81bbcd30b68d9cc6d738d10375c05622ce3 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 07:30:47 +0800 Subject: [PATCH 045/105] [maven-release-plugin] prepare release prerender-java-1.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index eee3653..02e374a 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6-SNAPSHOT + 1.6 jar prerender-java From b9513f5e24386505476c5cc6edf4fe91156f6d57 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 07:31:01 +0800 Subject: [PATCH 046/105] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 02e374a..1f13612 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6 + 1.7-SNAPSHOT jar prerender-java From eae602ad6a8884a6659814874cb168cec802ba4c Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 07:41:22 +0800 Subject: [PATCH 047/105] read me --- README.md | 2 +- demo/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b9a7d5f..2af5988 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.4 + 1.6 ## How it works diff --git a/demo/pom.xml b/demo/pom.xml index 45f5542..1f9b5e4 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,7 +11,7 @@ com.github.greengerong prerender-java - 1.4 + 1.6 From a4358be1df805c9472fdb6b014848a6dc399b283 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 20:08:00 +0800 Subject: [PATCH 048/105] make it quick fail --- .../greengerong/PreRenderSEOFilter.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 9b1e296..41e4812 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -232,21 +232,6 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI return false; } - if (hasEscapedFragment(request)) { - log.trace("Request Has _escaped_fragment_; intercept: yes"); - return true; - } - - if (StringUtils.isBlank(userAgent)) { - log.trace("Request has blank userAgent; intercept: no"); - return false; - } - - if (!isInSearchUserAgent(userAgent)) { - log.trace("Request User-Agent is not a search bot; intercept: no"); - return false; - } - if (isInResources(url)) { log.trace("request is for a (static) resource; intercept: no"); return false; @@ -264,6 +249,21 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI return false; } + if (hasEscapedFragment(request)) { + log.trace("Request Has _escaped_fragment_; intercept: yes"); + return true; + } + + if (StringUtils.isBlank(userAgent)) { + log.trace("Request has blank userAgent; intercept: no"); + return false; + } + + if (!isInSearchUserAgent(userAgent)) { + log.trace("Request User-Agent is not a search bot; intercept: no"); + return false; + } + log.trace("Defaulting to request intercept: yes"); return true; } From 97790c428fc88faeef6cd13acc88d74aef8133ac Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 20:13:44 +0800 Subject: [PATCH 049/105] change version, do not too big --- pom.xml | 2 +- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1f13612..6da5c5a 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.7-SNAPSHOT + 1.7.1-SNAPSHOT jar prerender-java diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 41e4812..4350fa6 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -264,7 +264,7 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI return false; } - log.trace("Defaulting to request intercept: yes"); + log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); return true; } From e3891b44c1f3fa8e4a84e697d528fb2976f33d4f Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 20:16:40 +0800 Subject: [PATCH 050/105] [maven-release-plugin] prepare release prerender-java-1.7.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6da5c5a..828346b 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.7.1-SNAPSHOT + 1.7.1 jar prerender-java From ff9c108bea99be5b2f3ef370b2479f4076b57ad3 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 20:16:57 +0800 Subject: [PATCH 051/105] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 828346b..7fe2911 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.7.1 + 1.7.2-SNAPSHOT jar prerender-java From 7b69fd2fe7dc8d32e5df98c30444eb92effd14cd Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 20:32:59 +0800 Subject: [PATCH 052/105] change version to 1.6.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7fe2911..41a54f7 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.7.2-SNAPSHOT + 1.6.1-SNAPSHOT jar prerender-java From 5a61cc4bd040704be800be7286f8dc1319752c3b Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 20:38:26 +0800 Subject: [PATCH 053/105] [maven-release-plugin] prepare release prerender-java-1.6.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 41a54f7..0fc1438 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.1-SNAPSHOT + 1.6.1 jar prerender-java From ec6c8e51a0ea5eec1843562b93f83707f9cd6adb Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 20:38:45 +0800 Subject: [PATCH 054/105] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0fc1438..47e46f1 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.1 + 1.6.2-SNAPSHOT jar prerender-java From c0dcb1d19412b4e35aa6950e9563e48c3cf6c868 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 18 Feb 2014 20:55:17 +0800 Subject: [PATCH 055/105] new version --- README.md | 2 +- demo/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2af5988..7ae6af0 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.6 + 1.6.1 ## How it works diff --git a/demo/pom.xml b/demo/pom.xml index 1f9b5e4..813d096 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,7 +11,7 @@ com.github.greengerong prerender-java - 1.6 + 1.6.1 From 10e9af5f840e3c14d6d3def0ba1bfb8aca816d40 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Wed, 5 Mar 2014 01:46:31 +0800 Subject: [PATCH 056/105] 1: add log for demo. 2 add html to event(due to java stream con not read more) --- demo/pom.xml | 23 +++++- demo/src/main/resources/log4j.properties | 72 +++++++++++++++++++ demo/src/main/webapp/WEB-INF/web.xml | 4 ++ prerender-java.iml | 1 + .../greengerong/PreRenderEventHandler.java | 4 +- .../greengerong/PreRenderSEOFilter.java | 29 +++++--- 6 files changed, 120 insertions(+), 13 deletions(-) create mode 100644 demo/src/main/resources/log4j.properties diff --git a/demo/pom.xml b/demo/pom.xml index 813d096..825c95c 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,11 +11,32 @@ com.github.greengerong prerender-java - 1.6.1 + 1.6.2-SNAPSHOT + + + com.github.greengerong + prerender-java-redis + 0.0.1-SNAPSHOT + + + commons-io + commons-io + 2.4 + + + org.slf4j + slf4j-log4j12 + 1.7.5 + + + log4j + log4j + 1.2.17 demo + org.mortbay.jetty diff --git a/demo/src/main/resources/log4j.properties b/demo/src/main/resources/log4j.properties new file mode 100644 index 0000000..8138149 --- /dev/null +++ b/demo/src/main/resources/log4j.properties @@ -0,0 +1,72 @@ +#log4j.rootLogger=CONSOLE,FILE +log4j.rootLogger=ALL,CONSOLE,FILE +log4j.addivity.org.apache=true + + +# 应用于控制台 +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=INFO +log4j.appender.CONSOLE.Target=System.out +log4j.appender.CONSOLE.Encoding=GBK +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +# 每天新建日志 +log4j.appender.A1=org.apache.log4j.DailyRollingFileAppender +log4j.appender.A1.File=C:/log4j/log +log4j.appender.A1.Encoding=GBK +log4j.appender.A1.Threshold=DEBUG +log4j.appender.A1.DatePattern='.'yyyy-MM-dd +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L : %m%n + +#应用于文件 +log4j.appender.FILE=org.apache.log4j.FileAppender +log4j.appender.FILE.File=C:/log4j/file.log +log4j.appender.FILE.Append=false +log4j.appender.FILE.Encoding=GBK +log4j.appender.FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +# 应用于文件回滚 +log4j.appender.ROLLING_FILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLING_FILE.Threshold=ERROR +log4j.appender.ROLLING_FILE.File=rolling.log +log4j.appender.ROLLING_FILE.Append=true +log4j.appender.CONSOLE_FILE.Encoding=GBK +log4j.appender.ROLLING_FILE.MaxFileSize=10KB +log4j.appender.ROLLING_FILE.MaxBackupIndex=1 +log4j.appender.ROLLING_FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLING_FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +#自定义Appender +log4j.appender.im = net.cybercorlin.util.logger.appender.IMAppender +log4j.appender.im.host = mail.cybercorlin.net +log4j.appender.im.username = username +log4j.appender.im.password = password +log4j.appender.im.recipient = yyflyons@163.com +log4j.appender.im.layout=org.apache.log4j.PatternLayout +log4j.appender.im.layout.ConversionPattern =[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +#应用于socket +log4j.appender.SOCKET=org.apache.log4j.RollingFileAppender +log4j.appender.SOCKET.RemoteHost=localhost +log4j.appender.SOCKET.Port=5001 +log4j.appender.SOCKET.LocationInfo=true +# Set up for Log Facter 5 +log4j.appender.SOCKET.layout=org.apache.log4j.PatternLayout +log4j.appender.SOCET.layout.ConversionPattern=[start]%d{DATE}[DATE]%n%p[PRIORITY]%n%x[NDC]%n%t[THREAD]%n%c[CATEGORY]%n%m[MESSAGE]%n%n +# Log Factor 5 Appender +log4j.appender.LF5_APPENDER=org.apache.log4j.lf5.LF5Appender +log4j.appender.LF5_APPENDER.MaxNumberOfRecords=2000 + +# 发送日志给邮件 +log4j.appender.MAIL=org.apache.log4j.net.SMTPAppender +log4j.appender.MAIL.Threshold=FATAL +log4j.appender.MAIL.BufferSize=10 +log4j.appender.MAIL.From=yyflyons@163.com +log4j.appender.MAIL.SMTPHost=www.wusetu.com +log4j.appender.MAIL.Subject=Log4J Message +log4j.appender.MAIL.To=yyflyons@126.com +log4j.appender.MAIL.layout=org.apache.log4j.PatternLayout +log4j.appender.MAIL.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n diff --git a/demo/src/main/webapp/WEB-INF/web.xml b/demo/src/main/webapp/WEB-INF/web.xml index eadd7d5..c5e037b 100644 --- a/demo/src/main/webapp/WEB-INF/web.xml +++ b/demo/src/main/webapp/WEB-INF/web.xml @@ -15,6 +15,10 @@ crawlerUserAgents YahooSeeker + + preRenderEventHandler + com.github.greengerong.PrerenderRedisCache + prerender diff --git a/prerender-java.iml b/prerender-java.iml index cbd3c69..7b5c7a6 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -6,6 +6,7 @@ + diff --git a/src/main/java/com/github/greengerong/PreRenderEventHandler.java b/src/main/java/com/github/greengerong/PreRenderEventHandler.java index 9e0d74e..9669afb 100644 --- a/src/main/java/com/github/greengerong/PreRenderEventHandler.java +++ b/src/main/java/com/github/greengerong/PreRenderEventHandler.java @@ -9,5 +9,7 @@ public interface PreRenderEventHandler { String beforeRender(HttpServletRequest clientRequest); - void afterRender(HttpServletRequest clientRequest, HttpResponse prerenderResponse); + void afterRender(HttpServletRequest clientRequest, HttpResponse prerenderResponse, String html); + + void destroy(); } diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 4350fa6..8914574 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -10,6 +10,7 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.message.BasicHeader; import org.apache.http.message.HeaderGroup; +import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,7 +19,6 @@ import javax.servlet.http.HttpServletResponse; import java.io.Closeable; import java.io.IOException; -import java.io.OutputStream; import java.io.PrintWriter; import java.net.URI; import java.net.URISyntaxException; @@ -40,7 +40,6 @@ public class PreRenderSEOFilter implements Filter { public void init(FilterConfig filterConfig) throws ServletException { this.prerenderConfig = new PrerenderConfig(filterConfig); this.httpClient = getHttpClient(); - this.preRenderEventHandler = prerenderConfig.getEventHandler(); } protected CloseableHttpClient getHttpClient() { @@ -54,7 +53,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo final HttpServletRequest request = (HttpServletRequest) servletRequest; final HttpServletResponse response = (HttpServletResponse) servletResponse; if (shouldShowPrerenderedPage(request)) { - + this.preRenderEventHandler = prerenderConfig.getEventHandler(); if (beforeRender(request, response) || proxyPrerenderedPageResponse(request, response)) { return; } @@ -90,9 +89,9 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer try { proxyResponse = httpClient.execute(getMethod); if (proxyResponse.getStatusLine().getStatusCode() == HTTP_OK) { - afterRender(request, proxyResponse); copyResponseHeaders(proxyResponse, response); - copyResponseEntity(proxyResponse, response); + final String html = copyResponseEntity(proxyResponse, response); + afterRender(request, proxyResponse, html); return true; } } finally { @@ -105,9 +104,9 @@ protected HttpGet getHttpGet(String apiUrl) { return new HttpGet(apiUrl); } - private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse) { + private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse, String html) { if (preRenderEventHandler != null) { - preRenderEventHandler.afterRender(request, proxyResponse); + preRenderEventHandler.afterRender(request, proxyResponse, html); } } @@ -133,16 +132,20 @@ protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletRespon /** * Copy response body data (the entity) from the proxy to the servlet client. */ - protected void copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) throws IOException { + protected String copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) throws IOException { HttpEntity entity = proxyResponse.getEntity(); if (entity != null) { - OutputStream servletOutputStream = servletResponse.getOutputStream(); + PrintWriter printWriter = servletResponse.getWriter(); try { - entity.writeTo(servletOutputStream); + final String html = EntityUtils.toString(entity); + printWriter.write(html); + printWriter.flush(); + return html; } finally { - closeQuietly(servletOutputStream); + closeQuietly(printWriter); } } + return ""; } protected void closeQuietly(Closeable closeable) { @@ -217,6 +220,10 @@ private String getFullUrl(HttpServletRequest request) { @Override public void destroy() { prerenderConfig = null; + if (preRenderEventHandler != null) { + preRenderEventHandler.destroy(); + preRenderEventHandler = null; + } closeQuietly(httpClient); } From 1237fc128f1cba6e7b60af43d65dd12d73f2a98f Mon Sep 17 00:00:00 2001 From: greengerongg Date: Wed, 5 Mar 2014 01:54:39 +0800 Subject: [PATCH 057/105] [maven-release-plugin] prepare release prerender-java-1.6.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 47e46f1..fdbd6a4 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.2-SNAPSHOT + 1.6.2 jar prerender-java From c6ccb4a51b096bf41ef81c67ade43b49927793b7 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Wed, 5 Mar 2014 01:54:52 +0800 Subject: [PATCH 058/105] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fdbd6a4..b2eaf60 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.2 + 1.6.3-SNAPSHOT jar prerender-java From 102225191152fbed3b2172e0a3da557d07f716d1 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Wed, 5 Mar 2014 01:58:14 +0800 Subject: [PATCH 059/105] release for cache event 1.6.2 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ae6af0..1eb1c85 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.6.1 + 1.6.2 ## How it works From 3cd62fd1f06fe4589e7b607ebe32b5219c543b28 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Wed, 5 Mar 2014 09:52:28 +0800 Subject: [PATCH 060/105] change demo --- .gitignore | 1 + demo/pom.xml | 2 +- demo/src/main/resources/log4j.properties | 15 +++------------ prerender-java.iml | 2 +- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index a9aaddb..71d7773 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ target/ **/target/ **/.idea/ +*.iml diff --git a/demo/pom.xml b/demo/pom.xml index 825c95c..2552382 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,7 +11,7 @@ com.github.greengerong prerender-java - 1.6.2-SNAPSHOT + 1.6.2 com.github.greengerong diff --git a/demo/src/main/resources/log4j.properties b/demo/src/main/resources/log4j.properties index 8138149..3074ab0 100644 --- a/demo/src/main/resources/log4j.properties +++ b/demo/src/main/resources/log4j.properties @@ -1,5 +1,5 @@ #log4j.rootLogger=CONSOLE,FILE -log4j.rootLogger=ALL,CONSOLE,FILE +log4j.rootLogger=DEBUG,CONSOLE log4j.addivity.org.apache=true @@ -39,15 +39,6 @@ log4j.appender.ROLLING_FILE.MaxBackupIndex=1 log4j.appender.ROLLING_FILE.layout=org.apache.log4j.PatternLayout log4j.appender.ROLLING_FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n -#自定义Appender -log4j.appender.im = net.cybercorlin.util.logger.appender.IMAppender -log4j.appender.im.host = mail.cybercorlin.net -log4j.appender.im.username = username -log4j.appender.im.password = password -log4j.appender.im.recipient = yyflyons@163.com -log4j.appender.im.layout=org.apache.log4j.PatternLayout -log4j.appender.im.layout.ConversionPattern =[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n - #应用于socket log4j.appender.SOCKET=org.apache.log4j.RollingFileAppender log4j.appender.SOCKET.RemoteHost=localhost @@ -64,9 +55,9 @@ log4j.appender.LF5_APPENDER.MaxNumberOfRecords=2000 log4j.appender.MAIL=org.apache.log4j.net.SMTPAppender log4j.appender.MAIL.Threshold=FATAL log4j.appender.MAIL.BufferSize=10 -log4j.appender.MAIL.From=yyflyons@163.com +log4j.appender.MAIL.From= log4j.appender.MAIL.SMTPHost=www.wusetu.com log4j.appender.MAIL.Subject=Log4J Message -log4j.appender.MAIL.To=yyflyons@126.com +log4j.appender.MAIL.To= log4j.appender.MAIL.layout=org.apache.log4j.PatternLayout log4j.appender.MAIL.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n diff --git a/prerender-java.iml b/prerender-java.iml index 7b5c7a6..968be0a 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -4,9 +4,9 @@ + - From 9dd185200f2d7517751b3758f798924b513bb9e9 Mon Sep 17 00:00:00 2001 From: Shahar Talmi Date: Fri, 11 Apr 2014 01:11:35 +0300 Subject: [PATCH 061/105] add init parameter for forwarded url header important for servers behind reverse proxy that need the public url to be used for pre-rendering. we usually set the original url in an http header which is added by the reverse proxy (similar to the more standard x-forwarded-proto and x-forwarded-for) --- README.md | 3 ++ .../greengerong/PreRenderSEOFilter.java | 19 ++++++++--- .../github/greengerong/PrerenderConfig.java | 4 +++ .../greengerong/PreRenderSEOFilterTest.java | 33 +++++++++++++++++++ 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1eb1c85..6c67fc5 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,9 @@ example: someproxy,someproxy1 ### blacklist +### forwardedURLHeader +Important for servers behind reverse proxy that need the public url to be used for pre-rendering. +We usually set the original url in an http header which is added by the reverse proxy (similar to the more standard `x-forwarded-proto` and `x-forwarded-for`) ### Using your own prerender service diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 8914574..f4cb717 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -207,14 +207,23 @@ protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest } } + private String getRequestURL(HttpServletRequest request) { + if (prerenderConfig.getForwardedURLHeader() != null) { + String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); + if (url != null) { + return url; + } + } + return request.getRequestURL().toString(); + } + private String getFullUrl(HttpServletRequest request) { - final StringBuffer url = request.getRequestURL(); + final String url = getRequestURL(request); final String queryString = request.getQueryString(); if (queryString != null) { - url.append('?'); - url.append(queryString); + return url + '?' + queryString; } - return url.toString(); + return url; } @Override @@ -229,7 +238,7 @@ public void destroy() { private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { final String userAgent = request.getHeader("User-Agent"); - final String url = request.getRequestURL().toString(); + final String url = getRequestURL(request); final String referer = request.getHeader("Referer"); log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index c5daa05..38b810e 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -53,6 +53,10 @@ public String getPrerenderToken() { return filterConfig.getInitParameter("prerenderToken"); } + public String getForwardedURLHeader() { + return filterConfig.getInitParameter("forwardedURLHeader"); + } + public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly"); diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 1ac7521..fe35749 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -266,4 +266,37 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and verify(httpClient).execute(httpGet); verify(filterChain, never()).doFilter(servletRequest, servletResponse); } + + @Test + public void should_use_request_url_from_custom_header_if_available() throws Exception { + //given + when(filterConfig.getInitParameter("forwardedURLHeader")).thenReturn("X-Forwarded-URL"); + when(filterConfig.getInitParameter("whitelist")).thenReturn("http://my.public.domain.com/"); + when(filterConfig.getInitParameter("blacklist")).thenReturn("http://localhost/test"); + + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getHeader("X-Forwarded-URL")).thenReturn("http://my.public.domain.com/"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put("_escaped_fragment_", ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } } From 61ae780c5557fe814c97428c1d9c6740083680ae Mon Sep 17 00:00:00 2001 From: green Date: Tue, 15 Apr 2014 22:40:25 +1000 Subject: [PATCH 062/105] java doc update for local java 8 compile --- pom.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b2eaf60..bb62f67 100644 --- a/pom.xml +++ b/pom.xml @@ -114,8 +114,13 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.7 + 2.8.1 + + http://docs.oracel.com/javase/6/docs/api/ + http://docs.oracel.com/javaee/6/api/ + + ${java.home}/../bin/javadoc From 44a64fb0c0461d2461cf6f8e0974447e61e86e3d Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 15 Apr 2014 22:51:48 +1000 Subject: [PATCH 063/105] change back java doc --- pom.xml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index bb62f67..380a7a8 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,5 @@ - + 4.0.0 com.github.greengerong @@ -26,7 +27,7 @@ oss-parent 7 - + UTF-8 4.11 @@ -115,13 +116,6 @@ org.apache.maven.plugins maven-javadoc-plugin 2.8.1 - - - http://docs.oracel.com/javase/6/docs/api/ - http://docs.oracel.com/javaee/6/api/ - - ${java.home}/../bin/javadoc - attach-javadoc From 1b5f9ef98a2be2df63bfc808c2267613a0ac13f1 Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Wed, 28 May 2014 21:38:18 +0300 Subject: [PATCH 064/105] refactoring prerender service out of filter --- .../greengerong/PreRenderSEOFilter.java | 353 ++++++++++-------- .../greengerong/PreRenderSEOFilterTest.java | 28 +- 2 files changed, 208 insertions(+), 173 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index f4cb717..89270de 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -29,39 +29,200 @@ import static com.google.common.collect.FluentIterable.from; public class PreRenderSEOFilter implements Filter { + private PrerenderService prerenderService; - private final static Logger log = LoggerFactory.getLogger(PreRenderSEOFilter.class); + @Override + public void init(FilterConfig filterConfig) throws ServletException { + this.prerenderService = new PrerenderService(new PrerenderConfig(filterConfig)); + } + + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) + throws IOException, ServletException { + boolean isPrerendered = prerenderService.tryPrerender( + (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); + if (!isPrerendered) { + filterChain.doFilter(servletRequest, servletResponse); + } + } + + @Override + public void destroy() { + prerenderService.destroy(); + } + + protected void setPrerenderService(PrerenderService prerenderService) { + this.prerenderService = prerenderService; + } +} + +class PrerenderService { + + public static final int HTTP_OK = 200; private CloseableHttpClient httpClient; - private PreRenderEventHandler preRenderEventHandler; private PrerenderConfig prerenderConfig; - public static final int HTTP_OK = 200; + private PreRenderEventHandler preRenderEventHandler; - @Override - public void init(FilterConfig filterConfig) throws ServletException { - this.prerenderConfig = new PrerenderConfig(filterConfig); + /** + * These are the "hop-by-hop" headers that should not be copied. + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html + * I use an HttpClient HeaderGroup class instead of Set because this + * approach does case insensitive lookup faster. + */ + protected static final HeaderGroup hopByHopHeaders; + + public PrerenderService(PrerenderConfig prerenderConfig) { + this.prerenderConfig = prerenderConfig; this.httpClient = getHttpClient(); } - protected CloseableHttpClient getHttpClient() { - return prerenderConfig.getHttpClient(); + static { + hopByHopHeaders = new HeaderGroup(); + String[] headers = new String[]{ + "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", + "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; + for (String header : headers) { + hopByHopHeaders.addHeader(new BasicHeader(header, null)); + } } - @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) - throws IOException, ServletException { + public void destroy() { + if (preRenderEventHandler != null) { + preRenderEventHandler.destroy(); + } + closeQuietly(httpClient); + } + + public boolean tryPrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) { try { - final HttpServletRequest request = (HttpServletRequest) servletRequest; - final HttpServletResponse response = (HttpServletResponse) servletResponse; - if (shouldShowPrerenderedPage(request)) { - this.preRenderEventHandler = prerenderConfig.getEventHandler(); - if (beforeRender(request, response) || proxyPrerenderedPageResponse(request, response)) { - return; - } - } + if (handlePrerender(servletRequest, servletResponse)) + return true; } catch (Exception e) { log.error("Prerender service error", e); } - filterChain.doFilter(servletRequest, servletResponse); + return false; + } + + private boolean handlePrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) + throws URISyntaxException, IOException { + if (shouldShowPrerenderedPage(servletRequest)) { + this.preRenderEventHandler = prerenderConfig.getEventHandler(); + if (beforeRender(servletRequest, servletResponse) || proxyPrerenderedPageResponse(servletRequest, servletResponse)) { + return true; + } + } + return false; + } + + private boolean hasEscapedFragment(HttpServletRequest request) { + return request.getParameterMap().containsKey("_escaped_fragment_"); + } + + public String getApiUrl(String url) { + String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); + if (!prerenderServiceUrl.endsWith("/")) { + prerenderServiceUrl += "/"; + } + return prerenderServiceUrl + url; + } + + private boolean isInBlackList(final String url, final String referer, List blacklist) { + return from(blacklist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + final Pattern pattern = Pattern.compile(regex); + return pattern.matcher(url).matches() || + (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); + } + }); + } + + private boolean isInSearchUserAgent(final String userAgent) { + return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return userAgent.toLowerCase().indexOf(item.toLowerCase()) >= 0; + } + }); + } + + private final static Logger log = LoggerFactory.getLogger(PrerenderService.class); + + public boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { + final String userAgent = request.getHeader("User-Agent"); + final String url = getRequestURL(request); + final String referer = request.getHeader("Referer"); + + log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); + + if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { + log.trace("Request is not HTTP GET; intercept: no"); + return false; + } + + if (isInResources(url)) { + log.trace("request is for a (static) resource; intercept: no"); + return false; + } + + final List whiteList = prerenderConfig.getWhitelist(); + if (whiteList != null && !isInWhiteList(url, whiteList)) { + log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); + return false; + } + + final List blacklist = prerenderConfig.getBlacklist(); + if (blacklist != null && isInBlackList(url, referer, blacklist)) { + log.trace("Blacklist is enabled, and this request is listed; intercept: no"); + return false; + } + + if (hasEscapedFragment(request)) { + log.trace("Request Has _escaped_fragment_; intercept: yes"); + return true; + } + + if (StringUtils.isBlank(userAgent)) { + log.trace("Request has blank userAgent; intercept: no"); + return false; + } + + if (!isInSearchUserAgent(userAgent)) { + log.trace("Request User-Agent is not a search bot; intercept: no"); + return false; + } + + log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); + return true; + } + + + public String getRequestURL(HttpServletRequest request) { + if (prerenderConfig.getForwardedURLHeader() != null) { + String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); + if (url != null) { + return url; + } + } + return request.getRequestURL().toString(); + } + + private boolean isInResources(final String url) { + return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return url.contains(item.toLowerCase()); + } + }); + } + + private boolean isInWhiteList(final String url, List whitelist) { + return from(whitelist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + return Pattern.compile(regex).matcher(url).matches(); + } + }); } private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) throws IOException { @@ -100,10 +261,6 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer return false; } - protected HttpGet getHttpGet(String apiUrl) { - return new HttpGet(apiUrl); - } - private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse, String html) { if (preRenderEventHandler != null) { preRenderEventHandler.afterRender(request, proxyResponse, html); @@ -158,22 +315,13 @@ protected void closeQuietly(Closeable closeable) { } } - /** - * These are the "hop-by-hop" headers that should not be copied. - * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html - * I use an HttpClient HeaderGroup class instead of Set because this - * approach does case insensitive lookup faster. - */ - protected static final HeaderGroup hopByHopHeaders; - - static { - hopByHopHeaders = new HeaderGroup(); - String[] headers = new String[]{ - "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", - "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; - for (String header : headers) { - hopByHopHeaders.addHeader(new BasicHeader(header, null)); + private String getFullUrl(HttpServletRequest request) { + final String url = getRequestURL(request); + final String queryString = request.getQueryString(); + if (queryString != null) { + return url + '?' + queryString; } + return url; } /** @@ -207,131 +355,12 @@ protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest } } - private String getRequestURL(HttpServletRequest request) { - if (prerenderConfig.getForwardedURLHeader() != null) { - String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); - if (url != null) { - return url; - } - } - return request.getRequestURL().toString(); - } - - private String getFullUrl(HttpServletRequest request) { - final String url = getRequestURL(request); - final String queryString = request.getQueryString(); - if (queryString != null) { - return url + '?' + queryString; - } - return url; - } - - @Override - public void destroy() { - prerenderConfig = null; - if (preRenderEventHandler != null) { - preRenderEventHandler.destroy(); - preRenderEventHandler = null; - } - closeQuietly(httpClient); - } - - private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { - final String userAgent = request.getHeader("User-Agent"); - final String url = getRequestURL(request); - final String referer = request.getHeader("Referer"); - - log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); - - if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { - log.trace("Request is not HTTP GET; intercept: no"); - return false; - } - - if (isInResources(url)) { - log.trace("request is for a (static) resource; intercept: no"); - return false; - } - - final List whiteList = prerenderConfig.getWhitelist(); - if (whiteList != null && !isInWhiteList(url, whiteList)) { - log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); - return false; - } - - final List blacklist = prerenderConfig.getBlacklist(); - if (blacklist != null && isInBlackList(url, referer, blacklist)) { - log.trace("Blacklist is enabled, and this request is listed; intercept: no"); - return false; - } - - if (hasEscapedFragment(request)) { - log.trace("Request Has _escaped_fragment_; intercept: yes"); - return true; - } - - if (StringUtils.isBlank(userAgent)) { - log.trace("Request has blank userAgent; intercept: no"); - return false; - } - - if (!isInSearchUserAgent(userAgent)) { - log.trace("Request User-Agent is not a search bot; intercept: no"); - return false; - } - - log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); - return true; - } - - private boolean hasEscapedFragment(HttpServletRequest request) { - return request.getParameterMap().containsKey("_escaped_fragment_"); - } - - private String getApiUrl(String url) { - String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); - if (!prerenderServiceUrl.endsWith("/")) { - prerenderServiceUrl += "/"; - } - return prerenderServiceUrl + url; - } - private boolean isInBlackList(final String url, final String referer, List blacklist) { - return from(blacklist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - final Pattern pattern = Pattern.compile(regex); - return pattern.matcher(url).matches() || - (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); - } - }); - } - - private boolean isInWhiteList(final String url, List whitelist) { - return from(whitelist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - return Pattern.compile(regex).matcher(url).matches(); - } - }); - } - - private boolean isInResources(final String url) { - return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return url.contains(item.toLowerCase()); - } - }); + protected HttpGet getHttpGet(String apiUrl) { + return new HttpGet(apiUrl); } - private boolean isInSearchUserAgent(final String userAgent) { - return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return userAgent.toLowerCase().indexOf(item.toLowerCase()) >= 0; - } - }); + protected CloseableHttpClient getHttpClient() { + return prerenderConfig.getHttpClient(); } - } diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index fe35749..d33edff 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -15,6 +15,7 @@ import javax.servlet.FilterChain; import javax.servlet.FilterConfig; +import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.util.Enumeration; @@ -50,13 +51,18 @@ public class PreRenderSEOFilterTest { public void setUp() throws Exception { preRenderSEOFilter = new PreRenderSEOFilter() { @Override - protected CloseableHttpClient getHttpClient() { - return httpClient; - } - - @Override - protected HttpGet getHttpGet(String apiUrl) { - return httpGet; + public void init(FilterConfig filterConfig) throws ServletException { + setPrerenderService(new PrerenderService(new PrerenderConfig(filterConfig)) { + @Override + protected CloseableHttpClient getHttpClient() { + return httpClient; + } + + @Override + protected HttpGet getHttpGet(String apiUrl) { + return httpGet; + } + }); } }; } @@ -91,7 +97,7 @@ public void should_handle_when_url_with_escaped_fragment_() throws Exception { final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -193,7 +199,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when @@ -256,7 +262,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when @@ -289,7 +295,7 @@ public void should_use_request_url_from_custom_header_if_available() throws Exce final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when From b25cc9f8d87b1971bd327e8bdf146f4538079faa Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Wed, 28 May 2014 22:15:03 +0300 Subject: [PATCH 065/105] removed prerender config dependency on filter configuration --- .../greengerong/PreRenderSEOFilter.java | 351 +----------------- .../github/greengerong/PrerenderConfig.java | 28 +- .../greengerong/PrerenderSeoService.java | 339 +++++++++++++++++ .../greengerong/PreRenderSEOFilterTest.java | 10 +- 4 files changed, 376 insertions(+), 352 deletions(-) create mode 100644 src/main/java/com/github/greengerong/PrerenderSeoService.java diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 89270de..21963b1 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,45 +1,30 @@ package com.github.greengerong; - -import com.google.common.base.Predicate; -import org.apache.commons.lang.StringUtils; -import org.apache.http.*; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIUtils; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.message.BasicHeader; -import org.apache.http.message.HeaderGroup; -import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.google.common.collect.Lists; import javax.servlet.*; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import java.io.Closeable; import java.io.IOException; -import java.io.PrintWriter; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Enumeration; +import java.util.HashMap; import java.util.List; -import java.util.regex.Pattern; - -import static com.google.common.collect.FluentIterable.from; +import java.util.Map; public class PreRenderSEOFilter implements Filter { - private PrerenderService prerenderService; + public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", + "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", + "blacklist", "prerenderServiceUrl"); + private PrerenderSeoService prerenderSeoService; @Override public void init(FilterConfig filterConfig) throws ServletException { - this.prerenderService = new PrerenderService(new PrerenderConfig(filterConfig)); + this.prerenderSeoService = new PrerenderSeoService(new PrerenderConfig(toMap(filterConfig))); } @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - boolean isPrerendered = prerenderService.tryPrerender( + boolean isPrerendered = prerenderSeoService.tryPrerender( (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); if (!isPrerendered) { filterChain.doFilter(servletRequest, servletResponse); @@ -48,319 +33,19 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo @Override public void destroy() { - prerenderService.destroy(); - } - - protected void setPrerenderService(PrerenderService prerenderService) { - this.prerenderService = prerenderService; - } -} - -class PrerenderService { - - public static final int HTTP_OK = 200; - private CloseableHttpClient httpClient; - private PrerenderConfig prerenderConfig; - private PreRenderEventHandler preRenderEventHandler; - - /** - * These are the "hop-by-hop" headers that should not be copied. - * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html - * I use an HttpClient HeaderGroup class instead of Set because this - * approach does case insensitive lookup faster. - */ - protected static final HeaderGroup hopByHopHeaders; - - public PrerenderService(PrerenderConfig prerenderConfig) { - this.prerenderConfig = prerenderConfig; - this.httpClient = getHttpClient(); - } - - static { - hopByHopHeaders = new HeaderGroup(); - String[] headers = new String[]{ - "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", - "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; - for (String header : headers) { - hopByHopHeaders.addHeader(new BasicHeader(header, null)); - } - } - - public void destroy() { - if (preRenderEventHandler != null) { - preRenderEventHandler.destroy(); - } - closeQuietly(httpClient); - } - - public boolean tryPrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) { - try { - if (handlePrerender(servletRequest, servletResponse)) - return true; - } catch (Exception e) { - log.error("Prerender service error", e); - } - return false; - } - - private boolean handlePrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) - throws URISyntaxException, IOException { - if (shouldShowPrerenderedPage(servletRequest)) { - this.preRenderEventHandler = prerenderConfig.getEventHandler(); - if (beforeRender(servletRequest, servletResponse) || proxyPrerenderedPageResponse(servletRequest, servletResponse)) { - return true; - } - } - return false; - } - - private boolean hasEscapedFragment(HttpServletRequest request) { - return request.getParameterMap().containsKey("_escaped_fragment_"); - } - - public String getApiUrl(String url) { - String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); - if (!prerenderServiceUrl.endsWith("/")) { - prerenderServiceUrl += "/"; - } - return prerenderServiceUrl + url; - } - - private boolean isInBlackList(final String url, final String referer, List blacklist) { - return from(blacklist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - final Pattern pattern = Pattern.compile(regex); - return pattern.matcher(url).matches() || - (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); - } - }); - } - - private boolean isInSearchUserAgent(final String userAgent) { - return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return userAgent.toLowerCase().indexOf(item.toLowerCase()) >= 0; - } - }); - } - - private final static Logger log = LoggerFactory.getLogger(PrerenderService.class); - - public boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { - final String userAgent = request.getHeader("User-Agent"); - final String url = getRequestURL(request); - final String referer = request.getHeader("Referer"); - - log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); - - if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { - log.trace("Request is not HTTP GET; intercept: no"); - return false; - } - - if (isInResources(url)) { - log.trace("request is for a (static) resource; intercept: no"); - return false; - } - - final List whiteList = prerenderConfig.getWhitelist(); - if (whiteList != null && !isInWhiteList(url, whiteList)) { - log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); - return false; - } - - final List blacklist = prerenderConfig.getBlacklist(); - if (blacklist != null && isInBlackList(url, referer, blacklist)) { - log.trace("Blacklist is enabled, and this request is listed; intercept: no"); - return false; - } - - if (hasEscapedFragment(request)) { - log.trace("Request Has _escaped_fragment_; intercept: yes"); - return true; - } - - if (StringUtils.isBlank(userAgent)) { - log.trace("Request has blank userAgent; intercept: no"); - return false; - } - - if (!isInSearchUserAgent(userAgent)) { - log.trace("Request User-Agent is not a search bot; intercept: no"); - return false; - } - - log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); - return true; - } - - - public String getRequestURL(HttpServletRequest request) { - if (prerenderConfig.getForwardedURLHeader() != null) { - String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); - if (url != null) { - return url; - } - } - return request.getRequestURL().toString(); - } - - private boolean isInResources(final String url) { - return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return url.contains(item.toLowerCase()); - } - }); - } - - private boolean isInWhiteList(final String url, List whitelist) { - return from(whitelist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - return Pattern.compile(regex).matcher(url).matches(); - } - }); - } - - private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) throws IOException { - if (preRenderEventHandler != null) { - final String html = preRenderEventHandler.beforeRender(request); - if (StringUtils.isNotBlank(html)) { - final PrintWriter writer = response.getWriter(); - writer.write(html); - writer.flush(); - closeQuietly(writer); - return true; - } - } - return false; - } - - private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException { - final String apiUrl = getApiUrl(getFullUrl(request)); - log.trace(String.format("Prerender proxy will send request to:%s", apiUrl)); - final HttpGet getMethod = getHttpGet(apiUrl); - copyRequestHeaders(request, getMethod); - withPrerenderToken(getMethod); - CloseableHttpResponse proxyResponse = null; - - try { - proxyResponse = httpClient.execute(getMethod); - if (proxyResponse.getStatusLine().getStatusCode() == HTTP_OK) { - copyResponseHeaders(proxyResponse, response); - final String html = copyResponseEntity(proxyResponse, response); - afterRender(request, proxyResponse, html); - return true; - } - } finally { - closeQuietly(proxyResponse); - } - return false; + prerenderSeoService.destroy(); } - private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse, String html) { - if (preRenderEventHandler != null) { - preRenderEventHandler.afterRender(request, proxyResponse, html); - } - } - - private void withPrerenderToken(HttpRequest proxyRequest) { - final String token = prerenderConfig.getPrerenderToken(); - //for new version prerender with token. - if (StringUtils.isNotBlank(token)) { - proxyRequest.addHeader("X-Prerender-Token", token); - } - } - - /** - * Copy proxied response headers back to the servlet client. - */ - protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { - for (Header header : proxyResponse.getAllHeaders()) { - if (!hopByHopHeaders.containsHeader(header.getName())) { - servletResponse.addHeader(header.getName(), header.getValue()); - } - } - } - - /** - * Copy response body data (the entity) from the proxy to the servlet client. - */ - protected String copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) throws IOException { - HttpEntity entity = proxyResponse.getEntity(); - if (entity != null) { - PrintWriter printWriter = servletResponse.getWriter(); - try { - final String html = EntityUtils.toString(entity); - printWriter.write(html); - printWriter.flush(); - return html; - } finally { - closeQuietly(printWriter); - } - } - return ""; - } - - protected void closeQuietly(Closeable closeable) { - try { - if (closeable != null) { - closeable.close(); - } - } catch (IOException e) { - log.error("Close proxy error", e); - } - } - - private String getFullUrl(HttpServletRequest request) { - final String url = getRequestURL(request); - final String queryString = request.getQueryString(); - if (queryString != null) { - return url + '?' + queryString; - } - return url; + protected void setPrerenderSeoService(PrerenderSeoService prerenderSeoService) { + this.prerenderSeoService = prerenderSeoService; } - /** - * Copy request headers from the servlet client to the proxy request. - * - * @throws URISyntaxException - */ - protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) throws URISyntaxException { - // Get an Enumeration of all of the header names sent by the client - Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); - while (enumerationOfHeaderNames.hasMoreElements()) { - String headerName = (String) enumerationOfHeaderNames.nextElement(); - //Instead the content-length is effectively set via InputStreamEntity - if (!headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { - Enumeration headers = servletRequest.getHeaders(headerName); - while (headers.hasMoreElements()) {//sometimes more than one value - String headerValue = (String) headers.nextElement(); - // In case the proxy host is running multiple virtual servers, - // rewrite the Host header to ensure that we get content from - // the correct virtual server - if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { - HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); - headerValue = host.getHostName(); - if (host.getPort() != -1) { - headerValue += ":" + host.getPort(); - } - } - proxyRequest.addHeader(headerName, headerValue); - } - } + protected Map toMap(FilterConfig filterConfig) { + HashMap config = new HashMap(); + for (String parameterName : PARAMETER_NAMES) { + config.put(parameterName, filterConfig.getInitParameter(parameterName)); } - } - - - protected HttpGet getHttpGet(String apiUrl) { - return new HttpGet(apiUrl); - } - - protected CloseableHttpClient getHttpClient() { - return prerenderConfig.getHttpClient(); + return config; } } + diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 38b810e..3407848 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -12,20 +12,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.FilterConfig; import java.util.Arrays; import java.util.List; +import java.util.Map; public class PrerenderConfig { private final static Logger log = LoggerFactory.getLogger(PrerenderConfig.class); - private FilterConfig filterConfig; + private Map config; - public PrerenderConfig(FilterConfig filterConfig) { - this.filterConfig = filterConfig; + public PrerenderConfig(Map config) { + this.config = config; } public PreRenderEventHandler getEventHandler() { - final String preRenderEventHandler = filterConfig.getInitParameter("preRenderEventHandler"); + final String preRenderEventHandler = config.get("preRenderEventHandler"); if (StringUtils.isNotBlank(preRenderEventHandler)) { try { return (PreRenderEventHandler) Class.forName(preRenderEventHandler).newInstance(); @@ -38,9 +38,9 @@ public PreRenderEventHandler getEventHandler() { public CloseableHttpClient getHttpClient() { HttpClientBuilder builder = HttpClients.custom(); - final String proxy = filterConfig.getInitParameter("proxy"); + final String proxy = config.get("proxy"); if (StringUtils.isNotBlank(proxy)) { - final int proxyPort = Integer.parseInt(filterConfig.getInitParameter("proxyPort")); + final int proxyPort = Integer.parseInt(config.get("proxyPort")); DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); builder = builder.setRoutePlanner(routePlanner); } @@ -50,17 +50,17 @@ public CloseableHttpClient getHttpClient() { } public String getPrerenderToken() { - return filterConfig.getInitParameter("prerenderToken"); + return config.get("prerenderToken"); } public String getForwardedURLHeader() { - return filterConfig.getInitParameter("forwardedURLHeader"); + return config.get("forwardedURLHeader"); } public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly"); - final String crawlerUserAgentsFromConfig = filterConfig.getInitParameter("crawlerUserAgents"); + final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); if (StringUtils.isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); } @@ -73,7 +73,7 @@ public List getExtensionsToIgnore() { ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent"); - final String extensionsToIgnoreFromConfig = filterConfig.getInitParameter("extensionsToIgnore"); + final String extensionsToIgnoreFromConfig = config.get("extensionsToIgnore"); if (StringUtils.isNotBlank(extensionsToIgnoreFromConfig)) { extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); } @@ -82,7 +82,7 @@ public List getExtensionsToIgnore() { } public List getWhitelist() { - final String whitelist = filterConfig.getInitParameter("whitelist"); + final String whitelist = config.get("whitelist"); if (StringUtils.isNotBlank(whitelist)) { return Arrays.asList(whitelist.trim().split(",")); } @@ -90,7 +90,7 @@ public List getWhitelist() { } public List getBlacklist() { - final String blacklist = filterConfig.getInitParameter("blacklist"); + final String blacklist = config.get("blacklist"); if (StringUtils.isNotBlank(blacklist)) { return Arrays.asList(blacklist.trim().split(",")); } @@ -98,7 +98,7 @@ public List getBlacklist() { } public String getPrerenderServiceUrl() { - final String prerenderServiceUrl = filterConfig.getInitParameter("prerenderServiceUrl"); + final String prerenderServiceUrl = config.get("prerenderServiceUrl"); return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : "http://service.prerender.io/"; } } diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java new file mode 100644 index 0000000..79f6a61 --- /dev/null +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -0,0 +1,339 @@ +package com.github.greengerong; + +import com.google.common.base.Predicate; +import org.apache.commons.lang.StringUtils; +import org.apache.http.*; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIUtils; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicHeader; +import org.apache.http.message.HeaderGroup; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.Closeable; +import java.io.IOException; +import java.io.PrintWriter; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Enumeration; +import java.util.List; +import java.util.regex.Pattern; + +import static com.google.common.collect.FluentIterable.from; + +class PrerenderSeoService { + + public static final int HTTP_OK = 200; + private CloseableHttpClient httpClient; + private PrerenderConfig prerenderConfig; + private PreRenderEventHandler preRenderEventHandler; + + /** + * These are the "hop-by-hop" headers that should not be copied. + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html + * I use an HttpClient HeaderGroup class instead of Set because this + * approach does case insensitive lookup faster. + */ + protected static final HeaderGroup hopByHopHeaders; + + public PrerenderSeoService(PrerenderConfig prerenderConfig) { + this.prerenderConfig = prerenderConfig; + this.httpClient = getHttpClient(); + } + + static { + hopByHopHeaders = new HeaderGroup(); + String[] headers = new String[]{ + "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", + "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; + for (String header : headers) { + hopByHopHeaders.addHeader(new BasicHeader(header, null)); + } + } + + public void destroy() { + if (preRenderEventHandler != null) { + preRenderEventHandler.destroy(); + } + closeQuietly(httpClient); + } + + public boolean tryPrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) { + try { + if (handlePrerender(servletRequest, servletResponse)) + return true; + } catch (Exception e) { + log.error("Prerender service error", e); + } + return false; + } + + private boolean handlePrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) + throws URISyntaxException, IOException { + if (shouldShowPrerenderedPage(servletRequest)) { + this.preRenderEventHandler = prerenderConfig.getEventHandler(); + if (beforeRender(servletRequest, servletResponse) || proxyPrerenderedPageResponse(servletRequest, servletResponse)) { + return true; + } + } + return false; + } + + private boolean hasEscapedFragment(HttpServletRequest request) { + return request.getParameterMap().containsKey("_escaped_fragment_"); + } + + public String getApiUrl(String url) { + String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); + if (!prerenderServiceUrl.endsWith("/")) { + prerenderServiceUrl += "/"; + } + return prerenderServiceUrl + url; + } + + private boolean isInBlackList(final String url, final String referer, List blacklist) { + return from(blacklist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + final Pattern pattern = Pattern.compile(regex); + return pattern.matcher(url).matches() || + (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); + } + }); + } + + private boolean isInSearchUserAgent(final String userAgent) { + return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return userAgent.toLowerCase().contains(item.toLowerCase()); + } + }); + } + + private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); + + public boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { + final String userAgent = request.getHeader("User-Agent"); + final String url = getRequestURL(request); + final String referer = request.getHeader("Referer"); + + log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); + + if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { + log.trace("Request is not HTTP GET; intercept: no"); + return false; + } + + if (isInResources(url)) { + log.trace("request is for a (static) resource; intercept: no"); + return false; + } + + final List whiteList = prerenderConfig.getWhitelist(); + if (whiteList != null && !isInWhiteList(url, whiteList)) { + log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); + return false; + } + + final List blacklist = prerenderConfig.getBlacklist(); + if (blacklist != null && isInBlackList(url, referer, blacklist)) { + log.trace("Blacklist is enabled, and this request is listed; intercept: no"); + return false; + } + + if (hasEscapedFragment(request)) { + log.trace("Request Has _escaped_fragment_; intercept: yes"); + return true; + } + + if (StringUtils.isBlank(userAgent)) { + log.trace("Request has blank userAgent; intercept: no"); + return false; + } + + if (!isInSearchUserAgent(userAgent)) { + log.trace("Request User-Agent is not a search bot; intercept: no"); + return false; + } + + log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); + return true; + } + + + public String getRequestURL(HttpServletRequest request) { + if (prerenderConfig.getForwardedURLHeader() != null) { + String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); + if (url != null) { + return url; + } + } + return request.getRequestURL().toString(); + } + + private boolean isInResources(final String url) { + return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return url.contains(item.toLowerCase()); + } + }); + } + + private boolean isInWhiteList(final String url, List whitelist) { + return from(whitelist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + return Pattern.compile(regex).matcher(url).matches(); + } + }); + } + + private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) throws IOException { + if (preRenderEventHandler != null) { + final String html = preRenderEventHandler.beforeRender(request); + if (StringUtils.isNotBlank(html)) { + final PrintWriter writer = response.getWriter(); + writer.write(html); + writer.flush(); + closeQuietly(writer); + return true; + } + } + return false; + } + + private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) + throws IOException, URISyntaxException { + final String apiUrl = getApiUrl(getFullUrl(request)); + log.trace(String.format("Prerender proxy will send request to:%s", apiUrl)); + final HttpGet getMethod = getHttpGet(apiUrl); + copyRequestHeaders(request, getMethod); + withPrerenderToken(getMethod); + CloseableHttpResponse proxyResponse = null; + + try { + proxyResponse = httpClient.execute(getMethod); + if (proxyResponse.getStatusLine().getStatusCode() == HTTP_OK) { + copyResponseHeaders(proxyResponse, response); + final String html = copyResponseEntity(proxyResponse, response); + afterRender(request, proxyResponse, html); + return true; + } + } finally { + closeQuietly(proxyResponse); + } + return false; + } + + private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse, String html) { + if (preRenderEventHandler != null) { + preRenderEventHandler.afterRender(request, proxyResponse, html); + } + } + + private void withPrerenderToken(HttpRequest proxyRequest) { + final String token = prerenderConfig.getPrerenderToken(); + //for new version prerender with token. + if (StringUtils.isNotBlank(token)) { + proxyRequest.addHeader("X-Prerender-Token", token); + } + } + + /** + * Copy proxied response headers back to the servlet client. + */ + protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { + for (Header header : proxyResponse.getAllHeaders()) { + if (!hopByHopHeaders.containsHeader(header.getName())) { + servletResponse.addHeader(header.getName(), header.getValue()); + } + } + } + + /** + * Copy response body data (the entity) from the proxy to the servlet client. + */ + protected String copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) + throws IOException { + HttpEntity entity = proxyResponse.getEntity(); + if (entity != null) { + PrintWriter printWriter = servletResponse.getWriter(); + try { + final String html = EntityUtils.toString(entity); + printWriter.write(html); + printWriter.flush(); + return html; + } finally { + closeQuietly(printWriter); + } + } + return ""; + } + + protected void closeQuietly(Closeable closeable) { + try { + if (closeable != null) { + closeable.close(); + } + } catch (IOException e) { + log.error("Close proxy error", e); + } + } + + private String getFullUrl(HttpServletRequest request) { + final String url = getRequestURL(request); + final String queryString = request.getQueryString(); + if (queryString != null) { + return url + '?' + queryString; + } + return url; + } + + /** + * Copy request headers from the servlet client to the proxy request. + * + * @throws java.net.URISyntaxException + */ + protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) + throws URISyntaxException { + // Get an Enumeration of all of the header names sent by the client + Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); + while (enumerationOfHeaderNames.hasMoreElements()) { + String headerName = (String) enumerationOfHeaderNames.nextElement(); + //Instead the content-length is effectively set via InputStreamEntity + if (!headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { + Enumeration headers = servletRequest.getHeaders(headerName); + while (headers.hasMoreElements()) {//sometimes more than one value + String headerValue = (String) headers.nextElement(); + // In case the proxy host is running multiple virtual servers, + // rewrite the Host header to ensure that we get content from + // the correct virtual server + if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { + HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); + headerValue = host.getHostName(); + if (host.getPort() != -1) { + headerValue += ":" + host.getPort(); + } + } + proxyRequest.addHeader(headerName, headerValue); + } + } + } + } + + + protected HttpGet getHttpGet(String apiUrl) { + return new HttpGet(apiUrl); + } + + protected CloseableHttpClient getHttpClient() { + return prerenderConfig.getHttpClient(); + } +} diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index d33edff..5fb1dfa 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -52,7 +52,7 @@ public void setUp() throws Exception { preRenderSEOFilter = new PreRenderSEOFilter() { @Override public void init(FilterConfig filterConfig) throws ServletException { - setPrerenderService(new PrerenderService(new PrerenderConfig(filterConfig)) { + setPrerenderSeoService(new PrerenderSeoService(new PrerenderConfig(toMap(filterConfig))) { @Override protected CloseableHttpClient getHttpClient() { return httpClient; @@ -97,7 +97,7 @@ public void should_handle_when_url_with_escaped_fragment_() throws Exception { final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -199,7 +199,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when @@ -262,7 +262,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when @@ -295,7 +295,7 @@ public void should_use_request_url_from_custom_header_if_available() throws Exce final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when From 9803d6264a62e5da6f41639389c535f0b904499a Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Wed, 28 May 2014 22:30:21 +0300 Subject: [PATCH 066/105] reorganizing and better naming --- .../greengerong/PreRenderSEOFilter.java | 2 +- .../greengerong/PrerenderSeoService.java | 242 +++++++++--------- 2 files changed, 122 insertions(+), 122 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 21963b1..0984b9e 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -24,7 +24,7 @@ public void init(FilterConfig filterConfig) throws ServletException { @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - boolean isPrerendered = prerenderSeoService.tryPrerender( + boolean isPrerendered = prerenderSeoService.prerenderIfEligible( (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); if (!isPrerendered) { filterChain.doFilter(servletRequest, servletResponse); diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 79f6a61..45c60e1 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -27,6 +27,7 @@ import static com.google.common.collect.FluentIterable.from; class PrerenderSeoService { + private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); public static final int HTTP_OK = 200; private CloseableHttpClient httpClient; @@ -39,7 +40,7 @@ class PrerenderSeoService { * I use an HttpClient HeaderGroup class instead of Set because this * approach does case insensitive lookup faster. */ - protected static final HeaderGroup hopByHopHeaders; + private static final HeaderGroup hopByHopHeaders; public PrerenderSeoService(PrerenderConfig prerenderConfig) { this.prerenderConfig = prerenderConfig; @@ -63,10 +64,11 @@ public void destroy() { closeQuietly(httpClient); } - public boolean tryPrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) { + public boolean prerenderIfEligible(HttpServletRequest servletRequest, HttpServletResponse servletResponse) { try { - if (handlePrerender(servletRequest, servletResponse)) + if (handlePrerender(servletRequest, servletResponse)) { return true; + } } catch (Exception e) { log.error("Prerender service error", e); } @@ -84,41 +86,7 @@ private boolean handlePrerender(HttpServletRequest servletRequest, HttpServletRe return false; } - private boolean hasEscapedFragment(HttpServletRequest request) { - return request.getParameterMap().containsKey("_escaped_fragment_"); - } - - public String getApiUrl(String url) { - String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); - if (!prerenderServiceUrl.endsWith("/")) { - prerenderServiceUrl += "/"; - } - return prerenderServiceUrl + url; - } - - private boolean isInBlackList(final String url, final String referer, List blacklist) { - return from(blacklist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - final Pattern pattern = Pattern.compile(regex); - return pattern.matcher(url).matches() || - (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); - } - }); - } - - private boolean isInSearchUserAgent(final String userAgent) { - return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return userAgent.toLowerCase().contains(item.toLowerCase()); - } - }); - } - - private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); - - public boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { + private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { final String userAgent = request.getHeader("User-Agent"); final String url = getRequestURL(request); final String referer = request.getHeader("Referer"); @@ -166,8 +134,47 @@ public boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URIS return true; } + protected HttpGet getHttpGet(String apiUrl) { + return new HttpGet(apiUrl); + } + + protected CloseableHttpClient getHttpClient() { + return prerenderConfig.getHttpClient(); + } + + /** + * Copy request headers from the servlet client to the proxy request. + * + * @throws java.net.URISyntaxException + */ + private void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) + throws URISyntaxException { + // Get an Enumeration of all of the header names sent by the client + Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); + while (enumerationOfHeaderNames.hasMoreElements()) { + String headerName = (String) enumerationOfHeaderNames.nextElement(); + //Instead the content-length is effectively set via InputStreamEntity + if (!headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { + Enumeration headers = servletRequest.getHeaders(headerName); + while (headers.hasMoreElements()) {//sometimes more than one value + String headerValue = (String) headers.nextElement(); + // In case the proxy host is running multiple virtual servers, + // rewrite the Host header to ensure that we get content from + // the correct virtual server + if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { + HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); + headerValue = host.getHostName(); + if (host.getPort() != -1) { + headerValue += ":" + host.getPort(); + } + } + proxyRequest.addHeader(headerName, headerValue); + } + } + } + } - public String getRequestURL(HttpServletRequest request) { + private String getRequestURL(HttpServletRequest request) { if (prerenderConfig.getForwardedURLHeader() != null) { String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); if (url != null) { @@ -177,6 +184,81 @@ public String getRequestURL(HttpServletRequest request) { return request.getRequestURL().toString(); } + private String getApiUrl(String url) { + String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); + if (!prerenderServiceUrl.endsWith("/")) { + prerenderServiceUrl += "/"; + } + return prerenderServiceUrl + url; + } + + /** + * Copy proxied response headers back to the servlet client. + */ + private void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { + for (Header header : proxyResponse.getAllHeaders()) { + if (!hopByHopHeaders.containsHeader(header.getName())) { + servletResponse.addHeader(header.getName(), header.getValue()); + } + } + } + + /** + * Copy response body data (the entity) from the proxy to the servlet client. + */ + private String copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) + throws IOException { + HttpEntity entity = proxyResponse.getEntity(); + if (entity != null) { + PrintWriter printWriter = servletResponse.getWriter(); + try { + final String html = EntityUtils.toString(entity); + printWriter.write(html); + printWriter.flush(); + return html; + } finally { + closeQuietly(printWriter); + } + } + return ""; + } + + + protected void closeQuietly(Closeable closeable) { + try { + if (closeable != null) { + closeable.close(); + } + } catch (IOException e) { + log.error("Close proxy error", e); + } + } + + private boolean hasEscapedFragment(HttpServletRequest request) { + return request.getParameterMap().containsKey("_escaped_fragment_"); + } + + private boolean isInBlackList(final String url, final String referer, List blacklist) { + return from(blacklist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + final Pattern pattern = Pattern.compile(regex); + return pattern.matcher(url).matches() || + (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); + } + }); + } + + private boolean isInSearchUserAgent(final String userAgent) { + return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return userAgent.toLowerCase().contains(item.toLowerCase()); + } + }); + } + + private boolean isInResources(final String url) { return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { @Override @@ -246,47 +328,6 @@ private void withPrerenderToken(HttpRequest proxyRequest) { } } - /** - * Copy proxied response headers back to the servlet client. - */ - protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { - for (Header header : proxyResponse.getAllHeaders()) { - if (!hopByHopHeaders.containsHeader(header.getName())) { - servletResponse.addHeader(header.getName(), header.getValue()); - } - } - } - - /** - * Copy response body data (the entity) from the proxy to the servlet client. - */ - protected String copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) - throws IOException { - HttpEntity entity = proxyResponse.getEntity(); - if (entity != null) { - PrintWriter printWriter = servletResponse.getWriter(); - try { - final String html = EntityUtils.toString(entity); - printWriter.write(html); - printWriter.flush(); - return html; - } finally { - closeQuietly(printWriter); - } - } - return ""; - } - - protected void closeQuietly(Closeable closeable) { - try { - if (closeable != null) { - closeable.close(); - } - } catch (IOException e) { - log.error("Close proxy error", e); - } - } - private String getFullUrl(HttpServletRequest request) { final String url = getRequestURL(request); final String queryString = request.getQueryString(); @@ -295,45 +336,4 @@ private String getFullUrl(HttpServletRequest request) { } return url; } - - /** - * Copy request headers from the servlet client to the proxy request. - * - * @throws java.net.URISyntaxException - */ - protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) - throws URISyntaxException { - // Get an Enumeration of all of the header names sent by the client - Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); - while (enumerationOfHeaderNames.hasMoreElements()) { - String headerName = (String) enumerationOfHeaderNames.nextElement(); - //Instead the content-length is effectively set via InputStreamEntity - if (!headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { - Enumeration headers = servletRequest.getHeaders(headerName); - while (headers.hasMoreElements()) {//sometimes more than one value - String headerValue = (String) headers.nextElement(); - // In case the proxy host is running multiple virtual servers, - // rewrite the Host header to ensure that we get content from - // the correct virtual server - if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { - HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); - headerValue = host.getHostName(); - if (host.getPort() != -1) { - headerValue += ":" + host.getPort(); - } - } - proxyRequest.addHeader(headerName, headerValue); - } - } - } - } - - - protected HttpGet getHttpGet(String apiUrl) { - return new HttpGet(apiUrl); - } - - protected CloseableHttpClient getHttpClient() { - return prerenderConfig.getHttpClient(); - } } From a2ed50bbcf2f0946e9fdfef5665edbaee73cc2b4 Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Wed, 28 May 2014 22:34:56 +0300 Subject: [PATCH 067/105] rearrange class entries --- .../com/github/greengerong/PrerenderSeoService.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 45c60e1..141691c 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -27,13 +27,8 @@ import static com.google.common.collect.FluentIterable.from; class PrerenderSeoService { - private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); - public static final int HTTP_OK = 200; - private CloseableHttpClient httpClient; - private PrerenderConfig prerenderConfig; - private PreRenderEventHandler preRenderEventHandler; - + private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); /** * These are the "hop-by-hop" headers that should not be copied. * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html @@ -41,6 +36,9 @@ class PrerenderSeoService { * approach does case insensitive lookup faster. */ private static final HeaderGroup hopByHopHeaders; + private CloseableHttpClient httpClient; + private PrerenderConfig prerenderConfig; + private PreRenderEventHandler preRenderEventHandler; public PrerenderSeoService(PrerenderConfig prerenderConfig) { this.prerenderConfig = prerenderConfig; From b1563745eaca8d36ce9c44ee794e8c5aa4e82bbe Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Wed, 28 May 2014 22:48:50 +0300 Subject: [PATCH 068/105] simplified PrerenderSeoService construction --- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 2 +- .../java/com/github/greengerong/PrerenderSeoService.java | 5 +++-- .../java/com/github/greengerong/PreRenderSEOFilterTest.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 0984b9e..de582a1 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -18,7 +18,7 @@ public class PreRenderSEOFilter implements Filter { @Override public void init(FilterConfig filterConfig) throws ServletException { - this.prerenderSeoService = new PrerenderSeoService(new PrerenderConfig(toMap(filterConfig))); + this.prerenderSeoService = new PrerenderSeoService(toMap(filterConfig)); } @Override diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 141691c..7face17 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -22,6 +22,7 @@ import java.net.URISyntaxException; import java.util.Enumeration; import java.util.List; +import java.util.Map; import java.util.regex.Pattern; import static com.google.common.collect.FluentIterable.from; @@ -40,8 +41,8 @@ class PrerenderSeoService { private PrerenderConfig prerenderConfig; private PreRenderEventHandler preRenderEventHandler; - public PrerenderSeoService(PrerenderConfig prerenderConfig) { - this.prerenderConfig = prerenderConfig; + public PrerenderSeoService(Map config) { + this.prerenderConfig = new PrerenderConfig(config); this.httpClient = getHttpClient(); } diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 5fb1dfa..2945053 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -52,7 +52,7 @@ public void setUp() throws Exception { preRenderSEOFilter = new PreRenderSEOFilter() { @Override public void init(FilterConfig filterConfig) throws ServletException { - setPrerenderSeoService(new PrerenderSeoService(new PrerenderConfig(toMap(filterConfig))) { + setPrerenderSeoService(new PrerenderSeoService(toMap(filterConfig)) { @Override protected CloseableHttpClient getHttpClient() { return httpClient; From c798d1f5f143798d1dcf199bc68008917e789f7d Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Wed, 28 May 2014 23:54:52 +0300 Subject: [PATCH 069/105] add missing public keyword --- src/main/java/com/github/greengerong/PrerenderSeoService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 7face17..2a34683 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -27,7 +27,7 @@ import static com.google.common.collect.FluentIterable.from; -class PrerenderSeoService { +public class PrerenderSeoService { public static final int HTTP_OK = 200; private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); /** From c698f44a1996c356ea85900748d00398d94acdbf Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Tue, 17 Jun 2014 15:16:40 +0300 Subject: [PATCH 070/105] add configurable socket timeout to http client --- .../github/greengerong/PrerenderConfig.java | 23 +++++++++++++-- .../greengerong/PrerenderConfigTest.java | 28 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/github/greengerong/PrerenderConfigTest.java diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 3407848..fc566f5 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -4,6 +4,7 @@ import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; +import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; @@ -38,15 +39,33 @@ public PreRenderEventHandler getEventHandler() { public CloseableHttpClient getHttpClient() { HttpClientBuilder builder = HttpClients.custom(); + builder = configureProxy(builder); + builder = configureTimeout(builder); + builder = builder.setConnectionManager(new PoolingHttpClientConnectionManager()); + return builder.build(); + } + + private HttpClientBuilder configureProxy(HttpClientBuilder builder) { final String proxy = config.get("proxy"); if (StringUtils.isNotBlank(proxy)) { final int proxyPort = Integer.parseInt(config.get("proxyPort")); DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); builder = builder.setRoutePlanner(routePlanner); } + return builder; + } + + private HttpClientBuilder configureTimeout(HttpClientBuilder builder) { + final String socketTimeout = getSocketTimeout(); + if (socketTimeout != null) { + RequestConfig config = RequestConfig.custom().setSocketTimeout(Integer.parseInt(socketTimeout)).build(); + builder = builder.setDefaultRequestConfig(config); + } + return builder; + } - builder = builder.setConnectionManager(new PoolingHttpClientConnectionManager()); - return builder.build(); + public String getSocketTimeout() { + return config.get("socketTimeout"); } public String getPrerenderToken() { diff --git a/src/test/java/com/github/greengerong/PrerenderConfigTest.java b/src/test/java/com/github/greengerong/PrerenderConfigTest.java new file mode 100644 index 0000000..3371603 --- /dev/null +++ b/src/test/java/com/github/greengerong/PrerenderConfigTest.java @@ -0,0 +1,28 @@ +package com.github.greengerong; + +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class PrerenderConfigTest { + @Test(expected = NumberFormatException.class) + public void should_throw_exception_if_invalid_timeout_value_specified() throws Exception { + //given + Map configuration = new HashMap(); + configuration.put("socketTimeout", "not_an_int"); + PrerenderConfig config = new PrerenderConfig(configuration); + //when + config.getHttpClient(); + } + + @Test + public void should_pass_if_correct_timeout_value_specified() throws Exception { + //given + Map configuration = new HashMap(); + configuration.put("socketTimeout", "1000"); + PrerenderConfig config = new PrerenderConfig(configuration); + //when + config.getHttpClient(); + } +} From b6d954097427181294d51c353bf64d0206b4b2a7 Mon Sep 17 00:00:00 2001 From: greengerong Date: Tue, 17 Jun 2014 23:17:18 +0800 Subject: [PATCH 071/105] [(Fix) #11]: add support for prerender-status-code --- pom.xml | 5 +++-- prerender-java.iml | 1 - .../com/github/greengerong/PrerenderSeoService.java | 12 +++++------- .../github/greengerong/PreRenderSEOFilterTest.java | 6 ++++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pom.xml b/pom.xml index 380a7a8..842c22c 100644 --- a/pom.xml +++ b/pom.xml @@ -16,6 +16,7 @@ http://www.apache.org/licenses/LICENSE-2.0.txt repo + scm:git:git@github.com:greengerong/prerender-java.git @@ -91,12 +92,12 @@ sonatype-nexus-snapshots Sonatype Nexus Snapshots - http://oss.sonatype.org/content/repositories/snapshots + https://oss.sonatype.org/content/repositories/snapshots sonatype-nexus-staging Nexus Release Repository - http://oss.sonatype.org/service/local/staging/deploy/maven2/ + https://oss.sonatype.org/service/local/staging/deploy/maven2/ diff --git a/prerender-java.iml b/prerender-java.iml index 968be0a..cbd3c69 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -4,7 +4,6 @@ - diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 2a34683..54fbe39 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -301,16 +301,14 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer try { proxyResponse = httpClient.execute(getMethod); - if (proxyResponse.getStatusLine().getStatusCode() == HTTP_OK) { - copyResponseHeaders(proxyResponse, response); - final String html = copyResponseEntity(proxyResponse, response); - afterRender(request, proxyResponse, html); - return true; - } + response.setStatus(proxyResponse.getStatusLine().getStatusCode()); + copyResponseHeaders(proxyResponse, response); + final String html = copyResponseEntity(proxyResponse, response); + afterRender(request, proxyResponse, html); + return true; } finally { closeQuietly(proxyResponse); } - return false; } private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse, String html) { diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 2945053..6ae1935 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -211,7 +211,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and } @Test - public void should_not_handle_when_every_thing_is_ok_but_prerender_server_response_is_not_200() throws Exception { + public void should_handle_when_every_thing_is_ok_but_prerender_server_response_is_not_200() throws Exception { //given when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); preRenderSEOFilter.init(filterConfig); @@ -230,13 +230,15 @@ public void should_not_handle_when_every_thing_is_ok_but_prerender_server_respon map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(NOT_FOUND); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); //then verify(httpClient).execute(httpGet); - verify(filterChain).doFilter(servletRequest, servletResponse); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + verify(servletResponse).setStatus(NOT_FOUND); } From c8342fba58ed66d0f973d6becc7b62d83d4090de Mon Sep 17 00:00:00 2001 From: green Date: Tue, 17 Jun 2014 23:32:19 +0800 Subject: [PATCH 072/105] update version to 1.6.3-SNAPSHOT --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6c67fc5..19a1e53 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.6.2 + 1.6.3-SNAPSHOT ## How it works From 0b58e99c47ceb47edcc8733100eab59969adf548 Mon Sep 17 00:00:00 2001 From: "MD. Mohiuddin ahmed" Date: Sun, 13 Jul 2014 23:01:14 +0600 Subject: [PATCH 073/105] Added .xml in exclution list. Prerender renders xmls so Google SiteMap Test Fails because it gets www.testsite.com/sitemap.xml as an html and returns "Your Sitemap appears to be an HTML page. Please use a supported sitemap format instead". --- src/main/java/com/github/greengerong/PrerenderConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index fc566f5..16eeb6f 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -88,7 +88,7 @@ public List getCrawlerUserAgents() { } public List getExtensionsToIgnore() { - List extensionsToIgnore = Lists.newArrayList(".js", ".css", ".less", ".png", ".jpg", ".jpeg", + List extensionsToIgnore = Lists.newArrayList(".xml",".js", ".css", ".less", ".png", ".jpg", ".jpeg", ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent"); From 1507e84e27badb53cc75161f16051ebd655214c6 Mon Sep 17 00:00:00 2001 From: Richard Midwinter Date: Thu, 7 Aug 2014 21:20:17 +0100 Subject: [PATCH 074/105] Improved file extension checking. Updated logic to look for file extensions at the end of the URL, or just before the first question mark if one is present. This should prevent matching on subdomains or URL parameter values. --- src/main/java/com/github/greengerong/PrerenderSeoService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 54fbe39..e2e2d08 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -262,7 +262,8 @@ private boolean isInResources(final String url) { return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { @Override public boolean apply(String item) { - return url.contains(item.toLowerCase()); + return (url.indexOf('?') >= 0 ? url.substring(0, url.indexOf('?')) : url) + .toLowerCase().endsWith(item); } }); } From 8dd2cff8ea56f1a1765f8ba2afcea56db0260b1e Mon Sep 17 00:00:00 2001 From: greengerong Date: Wed, 13 Aug 2014 21:40:06 +0800 Subject: [PATCH 075/105] [maven-release-plugin] prepare release prerender-java-1.6.3 --- pom.xml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 842c22c..6abb782 100644 --- a/pom.xml +++ b/pom.xml @@ -1,10 +1,9 @@ - + 4.0.0 com.github.greengerong prerender-java - 1.6.3-SNAPSHOT + 1.6.3 jar prerender-java @@ -28,7 +27,7 @@ oss-parent 7 - + UTF-8 4.11 From 285d10f2a9c510dfe01623db672efaafd6488ad6 Mon Sep 17 00:00:00 2001 From: greengerong Date: Wed, 13 Aug 2014 21:40:19 +0800 Subject: [PATCH 076/105] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6abb782..112aab4 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.3 + 1.6.4-SNAPSHOT jar prerender-java From f3db14bf9e9f676a1e0f886a00a219a3744cdba2 Mon Sep 17 00:00:00 2001 From: greengerong Date: Sat, 16 Aug 2014 08:05:57 +0800 Subject: [PATCH 077/105] [(imporve) #21]: Getting jsessionids with urls. (Add response to event handler before reponse prerender html) --- .../greengerong/PreRenderEventHandler.java | 3 +- .../greengerong/PrerenderSeoService.java | 49 ++++++++++--------- .../greengerong/PreRenderSEOFilterTest.java | 12 ++++- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderEventHandler.java b/src/main/java/com/github/greengerong/PreRenderEventHandler.java index 9669afb..f1f6022 100644 --- a/src/main/java/com/github/greengerong/PreRenderEventHandler.java +++ b/src/main/java/com/github/greengerong/PreRenderEventHandler.java @@ -4,12 +4,13 @@ import org.apache.http.HttpResponse; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; public interface PreRenderEventHandler { String beforeRender(HttpServletRequest clientRequest); - void afterRender(HttpServletRequest clientRequest, HttpResponse prerenderResponse, String html); + void afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, HttpResponse prerenderResponse, String responseHtml); void destroy(); } diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index e2e2d08..176d128 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -202,24 +202,24 @@ private void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse } } + private String getResponseHtml(HttpResponse proxyResponse) + throws IOException { + HttpEntity entity = proxyResponse.getEntity(); + return entity != null ? EntityUtils.toString(entity) : ""; + } + /** * Copy response body data (the entity) from the proxy to the servlet client. */ - private String copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) + private void responseEntity(String html, HttpServletResponse servletResponse) throws IOException { - HttpEntity entity = proxyResponse.getEntity(); - if (entity != null) { - PrintWriter printWriter = servletResponse.getWriter(); - try { - final String html = EntityUtils.toString(entity); - printWriter.write(html); - printWriter.flush(); - return html; - } finally { - closeQuietly(printWriter); - } + PrintWriter printWriter = servletResponse.getWriter(); + try { + printWriter.write(html); + printWriter.flush(); + } finally { + closeQuietly(printWriter); } - return ""; } @@ -262,8 +262,8 @@ private boolean isInResources(final String url) { return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { @Override public boolean apply(String item) { - return (url.indexOf('?') >= 0 ? url.substring(0, url.indexOf('?')) : url) - .toLowerCase().endsWith(item); + return (url.indexOf('?') >= 0 ? url.substring(0, url.indexOf('?')) : url) + .toLowerCase().endsWith(item); } }); } @@ -298,23 +298,24 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer final HttpGet getMethod = getHttpGet(apiUrl); copyRequestHeaders(request, getMethod); withPrerenderToken(getMethod); - CloseableHttpResponse proxyResponse = null; + CloseableHttpResponse prerenderServerResponse = null; try { - proxyResponse = httpClient.execute(getMethod); - response.setStatus(proxyResponse.getStatusLine().getStatusCode()); - copyResponseHeaders(proxyResponse, response); - final String html = copyResponseEntity(proxyResponse, response); - afterRender(request, proxyResponse, html); + prerenderServerResponse = httpClient.execute(getMethod); + response.setStatus(prerenderServerResponse.getStatusLine().getStatusCode()); + copyResponseHeaders(prerenderServerResponse, response); + final String html = getResponseHtml(prerenderServerResponse); + afterRender(request, response, prerenderServerResponse, html); + responseEntity(html, response); return true; } finally { - closeQuietly(proxyResponse); + closeQuietly(prerenderServerResponse); } } - private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse, String html) { + private void afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, CloseableHttpResponse prerenderServerResponse, String responseHtml) { if (preRenderEventHandler != null) { - preRenderEventHandler.afterRender(request, proxyResponse, html); + preRenderEventHandler.afterRender(clientRequest, clientResponse, prerenderServerResponse, responseHtml); } } diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 6ae1935..e4c935d 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -18,6 +18,8 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.Enumeration; import java.util.HashMap; @@ -46,6 +48,8 @@ public class PreRenderSEOFilterTest { @Mock private HttpGet httpGet; + @Mock + private PrintWriter printWriter; @Before public void setUp() throws Exception { @@ -99,6 +103,8 @@ public void should_handle_when_url_with_escaped_fragment_() throws Exception { when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -201,7 +207,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); - + when(servletResponse.getWriter()).thenReturn(printWriter); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -231,6 +237,8 @@ public void should_handle_when_every_thing_is_ok_but_prerender_server_response_i when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(NOT_FOUND); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -266,6 +274,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -299,6 +308,7 @@ public void should_use_request_url_from_custom_header_if_available() throws Exce when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); From 3ce13c6f5eb840423088f6e9267fba7b4a5f94a2 Mon Sep 17 00:00:00 2001 From: greengerong Date: Sat, 16 Aug 2014 08:19:31 +0800 Subject: [PATCH 078/105] update version in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 19a1e53..7f71558 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.6.3-SNAPSHOT + 1.6.4-SNAPSHOT ## How it works From a27db6ba0c92a1c3e018f0e52f5c05de18993fc8 Mon Sep 17 00:00:00 2001 From: greengerong Date: Tue, 26 Aug 2014 22:56:51 +0800 Subject: [PATCH 079/105] [tech] response html can be changed by event handler after render --- .../com/github/greengerong/PreRenderEventHandler.java | 2 +- .../java/com/github/greengerong/PrerenderSeoService.java | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderEventHandler.java b/src/main/java/com/github/greengerong/PreRenderEventHandler.java index f1f6022..bd71634 100644 --- a/src/main/java/com/github/greengerong/PreRenderEventHandler.java +++ b/src/main/java/com/github/greengerong/PreRenderEventHandler.java @@ -10,7 +10,7 @@ public interface PreRenderEventHandler { String beforeRender(HttpServletRequest clientRequest); - void afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, HttpResponse prerenderResponse, String responseHtml); + String afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, HttpResponse prerenderResponse, String responseHtml); void destroy(); } diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 176d128..22445b9 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -304,8 +304,8 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer prerenderServerResponse = httpClient.execute(getMethod); response.setStatus(prerenderServerResponse.getStatusLine().getStatusCode()); copyResponseHeaders(prerenderServerResponse, response); - final String html = getResponseHtml(prerenderServerResponse); - afterRender(request, response, prerenderServerResponse, html); + String html = getResponseHtml(prerenderServerResponse); + html = afterRender(request, response, prerenderServerResponse, html); responseEntity(html, response); return true; } finally { @@ -313,10 +313,11 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer } } - private void afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, CloseableHttpResponse prerenderServerResponse, String responseHtml) { + private String afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, CloseableHttpResponse prerenderServerResponse, String responseHtml) { if (preRenderEventHandler != null) { - preRenderEventHandler.afterRender(clientRequest, clientResponse, prerenderServerResponse, responseHtml); + return preRenderEventHandler.afterRender(clientRequest, clientResponse, prerenderServerResponse, responseHtml); } + return responseHtml; } private void withPrerenderToken(HttpRequest proxyRequest) { From 7cd89b8cd1f2cb81a7e460815e749ee1549b5842 Mon Sep 17 00:00:00 2001 From: green Date: Mon, 29 Sep 2014 07:45:22 +0800 Subject: [PATCH 080/105] Update README.md --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7f71558..50d9bb9 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,8 @@ Use this java filter that prerenders a javascript-rendered page using an externa prerender com.github.greengerong.PreRenderSEOFilter - prerenderServiceUrl - http://localhost:3000 - - - crawlerUserAgents - me + prerenderToken + [get from prerender: https://prerender.io/] From 4febb8e481240f88aefb966bee79526ac8f5fba0 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 13 Nov 2014 14:28:42 +0800 Subject: [PATCH 081/105] [Tech] fix test, make the test to clean --- .../com/github/greengerong/PrerenderConfigTest.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/github/greengerong/PrerenderConfigTest.java b/src/test/java/com/github/greengerong/PrerenderConfigTest.java index 3371603..d144e85 100644 --- a/src/test/java/com/github/greengerong/PrerenderConfigTest.java +++ b/src/test/java/com/github/greengerong/PrerenderConfigTest.java @@ -1,12 +1,17 @@ package com.github.greengerong; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Test; import java.util.HashMap; import java.util.Map; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsNull.notNullValue; +import static org.junit.Assert.assertThat; + public class PrerenderConfigTest { - @Test(expected = NumberFormatException.class) + @Test(expected = Exception.class) public void should_throw_exception_if_invalid_timeout_value_specified() throws Exception { //given Map configuration = new HashMap(); @@ -23,6 +28,8 @@ public void should_pass_if_correct_timeout_value_specified() throws Exception { configuration.put("socketTimeout", "1000"); PrerenderConfig config = new PrerenderConfig(configuration); //when - config.getHttpClient(); + final CloseableHttpClient httpClient = config.getHttpClient(); + + assertThat(httpClient, is(notNullValue())); } } From d401f984252e81865a525cf35ca35457b3ba6914 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 13 Nov 2014 17:02:41 +0800 Subject: [PATCH 082/105] [Tech] clean code. --- .../greengerong/PreRenderSEOFilter.java | 3 +- .../github/greengerong/PrerenderConfig.java | 32 ++++++++------- .../greengerong/PrerenderSeoService.java | 39 ++++++++++++------- .../greengerong/PreRenderSEOFilterTest.java | 21 +++++----- 4 files changed, 55 insertions(+), 40 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index de582a1..6eb721f 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,6 +1,7 @@ package com.github.greengerong; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import javax.servlet.*; import javax.servlet.http.HttpServletRequest; @@ -41,7 +42,7 @@ protected void setPrerenderSeoService(PrerenderSeoService prerenderSeoService) { } protected Map toMap(FilterConfig filterConfig) { - HashMap config = new HashMap(); + Map config = Maps.newHashMap(); for (String parameterName : PARAMETER_NAMES) { config.put(parameterName, filterConfig.getInitParameter(parameterName)); } diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 16eeb6f..1219486 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -17,8 +17,11 @@ import java.util.List; import java.util.Map; +import static org.apache.commons.lang.StringUtils.isNotBlank; + public class PrerenderConfig { private final static Logger log = LoggerFactory.getLogger(PrerenderConfig.class); + public static final String PRERENDER_IO_SERVICE_URL = "http://service.prerender.io/"; private Map config; public PrerenderConfig(Map config) { @@ -27,7 +30,7 @@ public PrerenderConfig(Map config) { public PreRenderEventHandler getEventHandler() { final String preRenderEventHandler = config.get("preRenderEventHandler"); - if (StringUtils.isNotBlank(preRenderEventHandler)) { + if (isNotBlank(preRenderEventHandler)) { try { return (PreRenderEventHandler) Class.forName(preRenderEventHandler).newInstance(); } catch (Exception e) { @@ -38,19 +41,20 @@ public PreRenderEventHandler getEventHandler() { } public CloseableHttpClient getHttpClient() { - HttpClientBuilder builder = HttpClients.custom(); - builder = configureProxy(builder); - builder = configureTimeout(builder); - builder = builder.setConnectionManager(new PoolingHttpClientConnectionManager()); + HttpClientBuilder builder = HttpClients.custom() + .setConnectionManager(new PoolingHttpClientConnectionManager()); + + configureProxy(builder); + configureTimeout(builder); return builder.build(); } private HttpClientBuilder configureProxy(HttpClientBuilder builder) { final String proxy = config.get("proxy"); - if (StringUtils.isNotBlank(proxy)) { + if (isNotBlank(proxy)) { final int proxyPort = Integer.parseInt(config.get("proxyPort")); DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); - builder = builder.setRoutePlanner(routePlanner); + builder.setRoutePlanner(routePlanner); } return builder; } @@ -59,7 +63,7 @@ private HttpClientBuilder configureTimeout(HttpClientBuilder builder) { final String socketTimeout = getSocketTimeout(); if (socketTimeout != null) { RequestConfig config = RequestConfig.custom().setSocketTimeout(Integer.parseInt(socketTimeout)).build(); - builder = builder.setDefaultRequestConfig(config); + builder.setDefaultRequestConfig(config); } return builder; } @@ -80,7 +84,7 @@ public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly"); final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); - if (StringUtils.isNotBlank(crawlerUserAgentsFromConfig)) { + if (isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); } @@ -88,12 +92,12 @@ public List getCrawlerUserAgents() { } public List getExtensionsToIgnore() { - List extensionsToIgnore = Lists.newArrayList(".xml",".js", ".css", ".less", ".png", ".jpg", ".jpeg", + List extensionsToIgnore = Lists.newArrayList(".xml", ".js", ".css", ".less", ".png", ".jpg", ".jpeg", ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent"); final String extensionsToIgnoreFromConfig = config.get("extensionsToIgnore"); - if (StringUtils.isNotBlank(extensionsToIgnoreFromConfig)) { + if (isNotBlank(extensionsToIgnoreFromConfig)) { extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); } @@ -102,7 +106,7 @@ public List getExtensionsToIgnore() { public List getWhitelist() { final String whitelist = config.get("whitelist"); - if (StringUtils.isNotBlank(whitelist)) { + if (isNotBlank(whitelist)) { return Arrays.asList(whitelist.trim().split(",")); } return null; @@ -110,7 +114,7 @@ public List getWhitelist() { public List getBlacklist() { final String blacklist = config.get("blacklist"); - if (StringUtils.isNotBlank(blacklist)) { + if (isNotBlank(blacklist)) { return Arrays.asList(blacklist.trim().split(",")); } return null; @@ -118,6 +122,6 @@ public List getBlacklist() { public String getPrerenderServiceUrl() { final String prerenderServiceUrl = config.get("prerenderServiceUrl"); - return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : "http://service.prerender.io/"; + return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : PRERENDER_IO_SERVICE_URL; } } diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 22445b9..a59b697 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -1,5 +1,6 @@ package com.github.greengerong; +import com.google.common.base.Function; import com.google.common.base.Predicate; import org.apache.commons.lang.StringUtils; import org.apache.http.*; @@ -20,15 +21,18 @@ import java.io.PrintWriter; import java.net.URI; import java.net.URISyntaxException; +import java.util.Arrays; import java.util.Enumeration; import java.util.List; import java.util.Map; import java.util.regex.Pattern; import static com.google.common.collect.FluentIterable.from; +import static org.apache.commons.lang.StringUtils.isNotBlank; +import static org.apache.http.HttpHeaders.CONTENT_LENGTH; +import static org.apache.http.HttpHeaders.HOST; public class PrerenderSeoService { - public static final int HTTP_OK = 200; private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); /** * These are the "hop-by-hop" headers that should not be copied. @@ -37,6 +41,7 @@ public class PrerenderSeoService { * approach does case insensitive lookup faster. */ private static final HeaderGroup hopByHopHeaders; + public static final String ESCAPED_FRAGMENT_KEY = "_escaped_fragment_"; private CloseableHttpClient httpClient; private PrerenderConfig prerenderConfig; private PreRenderEventHandler preRenderEventHandler; @@ -90,7 +95,7 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI final String url = getRequestURL(request); final String referer = request.getHeader("Referer"); - log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); + log.trace(String.format("checking request for %s from User-Agent %s and referer %s", url, userAgent, referer)); if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { log.trace("Request is not HTTP GET; intercept: no"); @@ -153,14 +158,14 @@ private void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest p while (enumerationOfHeaderNames.hasMoreElements()) { String headerName = (String) enumerationOfHeaderNames.nextElement(); //Instead the content-length is effectively set via InputStreamEntity - if (!headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { + if (!headerName.equalsIgnoreCase(CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { Enumeration headers = servletRequest.getHeaders(headerName); while (headers.hasMoreElements()) {//sometimes more than one value String headerValue = (String) headers.nextElement(); // In case the proxy host is running multiple virtual servers, // rewrite the Host header to ensure that we get content from // the correct virtual server - if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { + if (headerName.equalsIgnoreCase(HOST)) { HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); headerValue = host.getHostName(); if (host.getPort() != -1) { @@ -194,12 +199,19 @@ private String getApiUrl(String url) { /** * Copy proxied response headers back to the servlet client. */ - private void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { - for (Header header : proxyResponse.getAllHeaders()) { - if (!hopByHopHeaders.containsHeader(header.getName())) { + private void copyResponseHeaders(HttpResponse proxyResponse, final HttpServletResponse servletResponse) { + from(Arrays.asList(proxyResponse.getAllHeaders())).filter(new Predicate
() { + @Override + public boolean apply(Header header) { + return !hopByHopHeaders.containsHeader(header.getName()); + } + }).transform(new Function() { + @Override + public Boolean apply(Header header) { servletResponse.addHeader(header.getName(), header.getValue()); + return true; } - } + }).toList(); } private String getResponseHtml(HttpResponse proxyResponse) @@ -234,7 +246,7 @@ protected void closeQuietly(Closeable closeable) { } private boolean hasEscapedFragment(HttpServletRequest request) { - return request.getParameterMap().containsKey("_escaped_fragment_"); + return request.getParameterMap().containsKey(ESCAPED_FRAGMENT_KEY); } private boolean isInBlackList(final String url, final String referer, List blacklist) { @@ -280,7 +292,7 @@ public boolean apply(String regex) { private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) throws IOException { if (preRenderEventHandler != null) { final String html = preRenderEventHandler.beforeRender(request); - if (StringUtils.isNotBlank(html)) { + if (isNotBlank(html)) { final PrintWriter writer = response.getWriter(); writer.write(html); writer.flush(); @@ -323,7 +335,7 @@ private String afterRender(HttpServletRequest clientRequest, HttpServletResponse private void withPrerenderToken(HttpRequest proxyRequest) { final String token = prerenderConfig.getPrerenderToken(); //for new version prerender with token. - if (StringUtils.isNotBlank(token)) { + if (isNotBlank(token)) { proxyRequest.addHeader("X-Prerender-Token", token); } } @@ -331,9 +343,6 @@ private void withPrerenderToken(HttpRequest proxyRequest) { private String getFullUrl(HttpServletRequest request) { final String url = getRequestURL(request); final String queryString = request.getQueryString(); - if (queryString != null) { - return url + '?' + queryString; - } - return url; + return isNotBlank(queryString) ? String.format("%s?%s", url, queryString) : url; } } diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index e4c935d..49229c2 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -19,10 +19,11 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.PrintWriter; -import java.io.StringWriter; import java.util.Enumeration; import java.util.HashMap; +import static com.github.greengerong.PrerenderSeoService.ESCAPED_FRAGMENT_KEY; +import static org.apache.http.HttpStatus.SC_OK; import static org.mockito.Mockito.*; @RunWith(MockitoJUnitRunner.class) @@ -99,9 +100,9 @@ public void should_handle_when_url_with_escaped_fragment_() throws Exception { when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(SC_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); when(servletResponse.getWriter()).thenReturn(printWriter); @@ -203,9 +204,9 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(SC_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); when(servletResponse.getWriter()).thenReturn(printWriter); //when @@ -233,7 +234,7 @@ public void should_handle_when_every_thing_is_ok_but_prerender_server_response_i when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(NOT_FOUND); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); @@ -270,9 +271,9 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(SC_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); when(servletResponse.getWriter()).thenReturn(printWriter); @@ -304,9 +305,9 @@ public void should_use_request_url_from_custom_header_if_available() throws Exce when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(SC_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); when(servletResponse.getWriter()).thenReturn(printWriter); From f011cc926c5105d38f0988ba8797f863699f9d5e Mon Sep 17 00:00:00 2001 From: Philipp Schumacher Date: Thu, 13 Nov 2014 11:07:20 +0100 Subject: [PATCH 083/105] introduce configuration for intercepting by default --- .../greengerong/PreRenderSEOFilter.java | 2 +- .../github/greengerong/PrerenderConfig.java | 8 +++++++ .../greengerong/PrerenderSeoService.java | 21 +++++++++++-------- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index de582a1..c9545e1 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -13,7 +13,7 @@ public class PreRenderSEOFilter implements Filter { public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl"); + "blacklist", "prerenderServiceUrl", "interceptByDefault"); private PrerenderSeoService prerenderSeoService; @Override diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 16eeb6f..f1aa3d5 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -75,6 +75,14 @@ public String getPrerenderToken() { public String getForwardedURLHeader() { return config.get("forwardedURLHeader"); } + + public boolean isInterceptByDefault() { + String interceptByDefault = config.get("interceptByDefault"); + if(interceptByDefault != null) { + return Boolean.parseBoolean(interceptByDefault); + } + return true; + } public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 22445b9..958fc04 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -119,18 +119,21 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI return true; } - if (StringUtils.isBlank(userAgent)) { - log.trace("Request has blank userAgent; intercept: no"); - return false; - } + boolean interceptByDefault = prerenderConfig.isInterceptByDefault(); + if(interceptByDefault) { + if (StringUtils.isBlank(userAgent)) { + log.trace("Request has blank userAgent; intercept: no"); + return false; + } - if (!isInSearchUserAgent(userAgent)) { - log.trace("Request User-Agent is not a search bot; intercept: no"); - return false; + if (!isInSearchUserAgent(userAgent)) { + log.trace("Request User-Agent is not a search bot; intercept: no"); + return false; + } } - log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); - return true; + log.trace(String.format("Defaulting to request intercept(user-agent=%s): %s", userAgent, interceptByDefault ? "yes" : "no")); + return interceptByDefault; } protected HttpGet getHttpGet(String apiUrl) { From bd3c3f5740130dadac37153f79323dfa7c578737 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 13 Nov 2014 22:44:55 +0800 Subject: [PATCH 084/105] [pull request #29] revert change, let's check it first. --- .../greengerong/PreRenderSEOFilter.java | 2 +- .../github/greengerong/PrerenderConfig.java | 8 ------- .../greengerong/PrerenderSeoService.java | 21 ++++++++----------- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 4477a2d..6eb721f 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -14,7 +14,7 @@ public class PreRenderSEOFilter implements Filter { public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl", "interceptByDefault"); + "blacklist", "prerenderServiceUrl"); private PrerenderSeoService prerenderSeoService; @Override diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 55b6dc7..1219486 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -79,14 +79,6 @@ public String getPrerenderToken() { public String getForwardedURLHeader() { return config.get("forwardedURLHeader"); } - - public boolean isInterceptByDefault() { - String interceptByDefault = config.get("interceptByDefault"); - if(interceptByDefault != null) { - return Boolean.parseBoolean(interceptByDefault); - } - return true; - } public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 66ae08c..a59b697 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -124,21 +124,18 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI return true; } - boolean interceptByDefault = prerenderConfig.isInterceptByDefault(); - if(interceptByDefault) { - if (StringUtils.isBlank(userAgent)) { - log.trace("Request has blank userAgent; intercept: no"); - return false; - } + if (StringUtils.isBlank(userAgent)) { + log.trace("Request has blank userAgent; intercept: no"); + return false; + } - if (!isInSearchUserAgent(userAgent)) { - log.trace("Request User-Agent is not a search bot; intercept: no"); - return false; - } + if (!isInSearchUserAgent(userAgent)) { + log.trace("Request User-Agent is not a search bot; intercept: no"); + return false; } - log.trace(String.format("Defaulting to request intercept(user-agent=%s): %s", userAgent, interceptByDefault ? "yes" : "no")); - return interceptByDefault; + log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); + return true; } protected HttpGet getHttpGet(String apiUrl) { From 78a94b575a29b5ec3514e368785537dde7a512a4 Mon Sep 17 00:00:00 2001 From: greengerong Date: Fri, 14 Nov 2014 12:44:39 +0800 Subject: [PATCH 085/105] [pull request #29] remove google, bing and yahoo from defaults crawler. add new crawler sync "https://github.com/prerender/prerender-node/blob/master/index.js#L32" --- src/main/java/com/github/greengerong/PrerenderConfig.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 1219486..e647631 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -81,8 +81,9 @@ public String getForwardedURLHeader() { } public List getCrawlerUserAgents() { - List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", - "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly"); + List crawlerUserAgents = Lists.newArrayList("baiduspider", + "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview" + , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot"); final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); if (isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); From 3c02d915634ed19c43d8d6be1917d916f6ffae98 Mon Sep 17 00:00:00 2001 From: greengerong Date: Fri, 14 Nov 2014 21:13:19 +0800 Subject: [PATCH 086/105] [Tech] clean code. --- .../greengerong/PreRenderSEOFilterTest.java | 25 ++++++++++--------- .../greengerong/PrerenderConfigTest.java | 6 ++--- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 49229c2..6776586 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -23,13 +23,14 @@ import java.util.HashMap; import static com.github.greengerong.PrerenderSeoService.ESCAPED_FRAGMENT_KEY; +import static org.apache.http.HttpStatus.SC_NOT_FOUND; import static org.apache.http.HttpStatus.SC_OK; +import static org.apache.http.client.methods.HttpGet.METHOD_NAME; import static org.mockito.Mockito.*; @RunWith(MockitoJUnitRunner.class) public class PreRenderSEOFilterTest { - public static final int NOT_FOUND = 404; private PreRenderSEOFilter preRenderSEOFilter; @Mock @@ -95,7 +96,7 @@ public void should_handle_when_url_with_escaped_fragment_() throws Exception { final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); @@ -120,7 +121,7 @@ public void should_not_handle_when_user_agent_is_not_crawler() throws Exception preRenderSEOFilter.init(filterConfig); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); when(servletRequest.getHeader("User-Agent")).thenReturn("no"); //when @@ -138,7 +139,7 @@ public void should_not_handle_when_url_is_a_resource() throws Exception { preRenderSEOFilter.init(filterConfig); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test.js")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); //when @@ -157,7 +158,7 @@ public void should_not_handle_when_white_list_is_not_empty_and_url_is_not_in_whi preRenderSEOFilter.init(filterConfig); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); //when @@ -176,7 +177,7 @@ public void should_not_handle_when_black_list_is_not_empty_and_url_is_in_black_l preRenderSEOFilter.init(filterConfig); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); //when @@ -197,7 +198,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); @@ -227,7 +228,7 @@ public void should_handle_when_every_thing_is_ok_but_prerender_server_response_i final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); @@ -236,7 +237,7 @@ public void should_handle_when_every_thing_is_ok_but_prerender_server_response_i final HashMap map = Maps.newHashMap(); map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(NOT_FOUND); + when(statusLine.getStatusCode()).thenReturn(SC_NOT_FOUND); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); when(servletResponse.getWriter()).thenReturn(printWriter); @@ -247,7 +248,7 @@ public void should_handle_when_every_thing_is_ok_but_prerender_server_response_i //then verify(httpClient).execute(httpGet); verify(filterChain, never()).doFilter(servletRequest, servletResponse); - verify(servletResponse).setStatus(NOT_FOUND); + verify(servletResponse).setStatus(SC_NOT_FOUND); } @@ -264,7 +265,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); @@ -298,7 +299,7 @@ public void should_use_request_url_from_custom_header_if_available() throws Exce final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeader("X-Forwarded-URL")).thenReturn("http://my.public.domain.com/"); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); diff --git a/src/test/java/com/github/greengerong/PrerenderConfigTest.java b/src/test/java/com/github/greengerong/PrerenderConfigTest.java index d144e85..531c021 100644 --- a/src/test/java/com/github/greengerong/PrerenderConfigTest.java +++ b/src/test/java/com/github/greengerong/PrerenderConfigTest.java @@ -1,9 +1,9 @@ package com.github.greengerong; +import com.google.common.collect.Maps; import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Test; -import java.util.HashMap; import java.util.Map; import static org.hamcrest.core.Is.is; @@ -14,7 +14,7 @@ public class PrerenderConfigTest { @Test(expected = Exception.class) public void should_throw_exception_if_invalid_timeout_value_specified() throws Exception { //given - Map configuration = new HashMap(); + Map configuration = Maps.newHashMap(); configuration.put("socketTimeout", "not_an_int"); PrerenderConfig config = new PrerenderConfig(configuration); //when @@ -24,7 +24,7 @@ public void should_throw_exception_if_invalid_timeout_value_specified() throws E @Test public void should_pass_if_correct_timeout_value_specified() throws Exception { //given - Map configuration = new HashMap(); + Map configuration = Maps.newHashMap(); configuration.put("socketTimeout", "1000"); PrerenderConfig config = new PrerenderConfig(configuration); //when From 01d0511e859458078379b258972a066be58483c6 Mon Sep 17 00:00:00 2001 From: Geir Engdahl Date: Mon, 26 Jan 2015 14:23:39 +0100 Subject: [PATCH 087/105] Encode the response body with the encoding used by the proxy response. --- .../greengerong/PrerenderSeoService.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index a59b697..250b075 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -200,6 +200,7 @@ private String getApiUrl(String url) { * Copy proxied response headers back to the servlet client. */ private void copyResponseHeaders(HttpResponse proxyResponse, final HttpServletResponse servletResponse) { + servletResponse.setCharacterEncoding(getContentCharSet(proxyResponse.getEntity())); from(Arrays.asList(proxyResponse.getAllHeaders())).filter(new Predicate
() { @Override public boolean apply(Header header) { @@ -213,6 +214,26 @@ public Boolean apply(Header header) { } }).toList(); } + + /** + * Get the charset used to encode the http entity. + */ + private String getContentCharSet(final HttpEntity entity) throws ParseException { + if (entity == null) { + return null; + } + String charset = null; + if (entity.getContentType() != null) { + HeaderElement values[] = entity.getContentType().getElements(); + if (values.length > 0) { + NameValuePair param = values[0].getParameterByName("charset"); + if (param != null) { + charset = param.getValue(); + } + } + } + return charset; + } private String getResponseHtml(HttpResponse proxyResponse) throws IOException { From 83d571e0d872d032f863c3ccc19489a1e8104cb9 Mon Sep 17 00:00:00 2001 From: Florian Hirsch Date: Thu, 1 Oct 2015 09:27:53 +0700 Subject: [PATCH 088/105] #34: Disabling auto-redirect-handling --- src/main/java/com/github/greengerong/PrerenderConfig.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index e647631..1dc1598 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -42,7 +42,8 @@ public PreRenderEventHandler getEventHandler() { public CloseableHttpClient getHttpClient() { HttpClientBuilder builder = HttpClients.custom() - .setConnectionManager(new PoolingHttpClientConnectionManager()); + .setConnectionManager(new PoolingHttpClientConnectionManager()) + .disableRedirectHandling(); configureProxy(builder); configureTimeout(builder); From c5fda9faefac4fbdfdb3466e0427f5f9c49a5b19 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 8 Oct 2015 20:20:28 +0800 Subject: [PATCH 089/105] readme & update prerender client --- README.md | 25 +++++++++++-------- demo/pom.xml | 10 ++++---- demo/src/main/webapp/WEB-INF/web.xml | 8 +++--- prerender-java.iml | 3 +-- .../github/greengerong/PrerenderConfig.java | 19 ++++++++------ 5 files changed, 37 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 50d9bb9..1affbe8 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa 2. Check to make sure we aren't requesting a resource (js, css, etc...) 3. (optional) Check to make sure the url is in the whitelist 4. (optional) Check to make sure the url isn't in the blacklist -2. Make a `GET` request to the [prerender service](https://github.com/collectiveip/prerender)(phantomjs server) for the page's prerendered HTML +2. Make a `GET` request to the [prerender service](https://github.com/prerender/prerender)(phantomjs server) for the page's prerendered HTML 3. Return that HTML to the crawler ## Customization @@ -56,7 +56,7 @@ We usually set the original url in an http header which is added by the reverse ### Using your own prerender service -If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://prerender.herokuapp.com` +If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://service.prerender.io/` $ export PRERENDER_SERVICE_URL= @@ -92,16 +92,21 @@ If you want to cache the caching, analytics, log or others, you can config it. I ## Testing -If you want to make sure your pages are rendering correctly: +If your URLs use a hash-bang: -1. Open the Developer Tools in Chrome (Cmd + Atl + J) -2. Click the Settings gear in the bottom right corner. -3. Click "Overrides" on the left side of the settings panel. -4. Check the "User Agent" checkbox. -6. Choose "Other..." from the User Agent dropdown. -7. Type `googlebot` into the input box. -8. Refresh the page (make sure to keep the developer tools open). + If you want to see `http://localhost:3000/#!/profiles/1234` + Then go to `http://localhost:3000/?_escaped_fragment_=/profiles/1234` +If your URLs use push-state: + + If you want to see `http://localhost:3000/profiles/1234` + Then go to `http://localhost:3000/profiles/1234?_escaped_fragment_=` + + +project demo test url: + + http://localhost:8080/test/?_escaped_fragment_= + ## License The MIT License (MIT) diff --git a/demo/pom.xml b/demo/pom.xml index 2552382..076a7bb 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -13,11 +13,11 @@ prerender-java 1.6.2 - - com.github.greengerong - prerender-java-redis - 0.0.1-SNAPSHOT - + + + + + commons-io commons-io diff --git a/demo/src/main/webapp/WEB-INF/web.xml b/demo/src/main/webapp/WEB-INF/web.xml index c5e037b..2677fca 100644 --- a/demo/src/main/webapp/WEB-INF/web.xml +++ b/demo/src/main/webapp/WEB-INF/web.xml @@ -15,10 +15,10 @@ crawlerUserAgents YahooSeeker - - preRenderEventHandler - com.github.greengerong.PrerenderRedisCache - + + + + prerender diff --git a/prerender-java.iml b/prerender-java.iml index cbd3c69..f74aa01 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -23,5 +23,4 @@ - - + \ No newline at end of file diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 1dc1598..90c2240 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -2,7 +2,6 @@ import com.google.common.collect.Lists; -import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; @@ -84,7 +83,8 @@ public String getForwardedURLHeader() { public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview" - , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot"); + , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot", "vkShare", + "W3C_Validator", "redditbot", "Applebot"); final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); if (isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); @@ -94,10 +94,10 @@ public List getCrawlerUserAgents() { } public List getExtensionsToIgnore() { - List extensionsToIgnore = Lists.newArrayList(".xml", ".js", ".css", ".less", ".png", ".jpg", ".jpeg", - ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", - ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", - ".iso", ".flv", ".m4v", ".torrent"); + List extensionsToIgnore = Lists.newArrayList(".js", ".json", ".css", ".xml", ".less", ".png", ".jpg", + ".jpeg", ".gif", ".pdf", ".doc", ".txt", ".ico", ".rss", ".zip", ".mp3", ".rar", ".exe", ".wmv", + ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", + ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent", ".woff", ".ttf"); final String extensionsToIgnoreFromConfig = config.get("extensionsToIgnore"); if (isNotBlank(extensionsToIgnoreFromConfig)) { extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); @@ -124,6 +124,11 @@ public List getBlacklist() { public String getPrerenderServiceUrl() { final String prerenderServiceUrl = config.get("prerenderServiceUrl"); - return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : PRERENDER_IO_SERVICE_URL; + return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : getDefaultPrerenderIoServiceUrl(); + } + + private String getDefaultPrerenderIoServiceUrl() { + final String prerenderServiceUrlInEnv = System.getProperty("PRERENDER_SERVICE_URL"); + return isNotBlank(prerenderServiceUrlInEnv) ? prerenderServiceUrlInEnv : PRERENDER_IO_SERVICE_URL; } } From 99cefef83e1b17041dc9f7067c607b9f288fabe8 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 8 Oct 2015 20:27:55 +0800 Subject: [PATCH 090/105] [maven-release-plugin] prepare release prerender-java-1.6.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 112aab4..251e979 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.4-SNAPSHOT + 1.6.4 jar prerender-java From d7b7d943208d973f6dde62be243753a1df4cc99f Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 8 Oct 2015 20:28:19 +0800 Subject: [PATCH 091/105] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 251e979..01017d3 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.4 + 1.6.5-SNAPSHOT jar prerender-java From 29a1e08027ae926fe10d3627d027b98c8b717696 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 8 Oct 2015 20:50:54 +0800 Subject: [PATCH 092/105] new version 1.6.4 --- README.md | 2 +- demo/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1affbe8..7d0af15 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.6.4-SNAPSHOT + 1.6.4 ## How it works diff --git a/demo/pom.xml b/demo/pom.xml index 076a7bb..5b1c40c 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,7 +11,7 @@ com.github.greengerong prerender-java - 1.6.2 + 1.6.4 From 707e199407debb385cd2b0683549ccd099dc5a61 Mon Sep 17 00:00:00 2001 From: Jan Philip Bernius Date: Fri, 6 May 2016 20:03:02 +0200 Subject: [PATCH 093/105] Remove .DS_Store files --- .DS_Store | Bin 6148 -> 0 bytes .gitignore | 3 +++ demo/.DS_Store | Bin 6148 -> 0 bytes demo/src/.DS_Store | Bin 6148 -> 0 bytes demo/src/main/.DS_Store | Bin 6148 -> 0 bytes demo/src/main/webapp/.DS_Store | Bin 6148 -> 0 bytes src/main/.DS_Store | Bin 6148 -> 0 bytes src/main/java/.DS_Store | Bin 6148 -> 0 bytes src/main/java/com/.DS_Store | Bin 6148 -> 0 bytes src/main/java/com/github/.DS_Store | Bin 6148 -> 0 bytes 10 files changed, 3 insertions(+) delete mode 100644 .DS_Store delete mode 100644 demo/.DS_Store delete mode 100644 demo/src/.DS_Store delete mode 100644 demo/src/main/.DS_Store delete mode 100644 demo/src/main/webapp/.DS_Store delete mode 100644 src/main/.DS_Store delete mode 100644 src/main/java/.DS_Store delete mode 100644 src/main/java/com/.DS_Store delete mode 100644 src/main/java/com/github/.DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index f8b64f14cb1b4abba656ea103bb350b269a79956..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKQA+|r5S~@5J1X!c@Nwx`LM>CFheHf{3>19GsJl*5Fi*TQAIv&}eoz0V-_y+Q zDw*{lghb82>^D0*bIW{p?A!u?sE+(RKpFrnRKkRf!xuv9q)U=8o)n^>XL#dq;PnCz z=dVlA?D&lg(7Q{44{f-E;N$!A2EA?=<#KN#kxXrDZdtZ%?X;TRz8{RM!Km+t7xhjs zu6O;m=MFEG9l6mcthz(Ba_&YQTv2u-2zqF@OmwsBN1dy5W-=)j3t5@3RHj)uDV1_r zd30EwPHoGw(=XQE{=vyv!+n@OKE;|lfuEU@U4t`tMq^&aeSO&P1^rv}QG60Y7{bl^ ziz^h1Sln^>CyFygd@5~LJ|i>03@`($F<=f;YkRdcj%Eg!fq%dNoevU~&@mWmR7VF6 z>I#5JzmZDNrn>~?NP~{SSR<~W2%UCFheHf{3>19GsJl*75Kp`_AIv&}eoz0V-_y+Q z8kq$?ghb82>^D0*bIW{p?A!u?s7?F=Kn4ITRKkRf!xuv9q$`pzo)n^>XM`hf5O_Fy z@kT4r?D&lg(7Q`Q2xGYQrXSzmpdUv0{HsVLQ(Nh6%eJlEcB?=1gJ~_84E^x3(F>-H zzTffO@s+Y8H=2YscdS+}+^B~u%5Db10PVJkZuR}Bcb&=3X2nt|CkrQ)c}~vC<$O*a zA64da+p_G;v$cP4czWJ+@8=JXv1U%-N2X-Y-~yh|Fh$%o#=}7{?4pOF(-47+#dV5|kqiItF8nxPl^dDxyvm=87S7I{KxFa}35Bbvg*OGd{=cEX)l> zsNK;oRX7MoBah4gGqB7+Qcp9~|Ifa^|1T%;j2U1C{uBcuRd3X5Sd!_jE5%W-m8dtU pBor5Gd{03`wPK8=R-8puf__N`qGK@Dh#nOF5l}Snzzl4Zfj5k^fwBMq diff --git a/demo/src/.DS_Store b/demo/src/.DS_Store deleted file mode 100644 index e66c6b5437e4c9df30bd8a4979ba83365488deeb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~K}*9h6vvaTYKscJ2s;MPMK*PA2M?u8@F;Y!gA;A3xI%5m+PTB57QxTsxAF7% zUXqH_okipYlK)HIdr9+yEeT_cd-JHnn9Ue7p`vm$))X9G>uTs=RE~3mi+ma;SqS|% z?fdJ3-)^vb7O#ZfW2wp+{PVYlmu&Pl&?#Io0GJL34LUzV0> zTK22Cw|{VYJ`5hqr{_v>)`ghzk? diff --git a/demo/src/main/.DS_Store b/demo/src/main/.DS_Store deleted file mode 100644 index 496035a1af584566e79dc9088a9c3c0ee69cbcd5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z-NTO;8~hp~uB@kyfqk!9yq&JPIlFU_}#BY#=0|NzI{FgW&V{Ha?Ft zyPHEWXAwICv)}Ce>}Eg6{xHV)c$NivHzj+ZCJPu-6qA=cDb` zGz`=FFpf`7uWqL9%l7rHQ2cSATFQ~e65he6Eall*MPVF0fxoONa|ww7Vt^PR26mVM zeJB_QJInzsofsen>KVZOL4YFK8ViMT>wpHY&lqnZqJWKW2}GgM)>tTn2nbiDfU1=1 zCk9vL;1@d2)>tT1<&4XjVH`bk`FP=ScJK?C&bY0RT4I10s4~!0T?^0uYxreqANkc1 z>JbCPz&~SvwyPE(St~#vK~XTTKm!Eq$|V3gxQ}#XP{jq>kY{Tw6yhl8 Rm*s$T5s-vXLk#=^1K(ORNofE8 diff --git a/demo/src/main/webapp/.DS_Store b/demo/src/main/webapp/.DS_Store deleted file mode 100644 index 2a74ced11ee55ebb91e09228cecc47288de92ca0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5T3Qwrl^pM(BtB{N~_lP;30%o^ddqFJy_9%hz*3MG^st*Y7l%L-^S;0 zW_Jr#>s7?g!0tCYKf9R^vOfSI+;LC^umQlrMkq*GB4l3bS~J0fLd|gw7jWuNlA#|h zndmQ?=-Ug>g%-H52cNz__R;g^52G+iE0uS#mdmei6s%&=+Uj+~K@d;ecsvM_t9C!0 zw8P-e_eR%BPrY=UxZX&4F1@sm9;J8UI6}Q=qPt;`_HXRcbb3&$m1Xs~F)PbyyyvNzobEX*-NVbw~gL8O6LHlM(7yKG^(uwJGwrTKTk-4I=v+br9sDFrV%|T zLZ>3?RAHVNLZ_o$nmET`rctMZP%Gm+X65$=PMPZV!OpgLU9=>D5x)70&qb4NJ%-hUy_b- Yj=@YL%|dpSj>s1QMF@Ayz%MZH0p1};X#fBK diff --git a/src/main/.DS_Store b/src/main/.DS_Store deleted file mode 100644 index 1310f12577816836235239bd947be7ffa0766e36..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~K}*9h6vvaTYKscJ2s;MPLpF782M?u8@F;Y!gA;A3xJqfq+HHreS_D6j-^S15 zdr2xzcNUQsNd7N*?V;Vgo?`1*idkIqpPNaQ8~^LEV4-urvdce zwC}GAe!IaQna}3T|N8yGB#yGY)q3Z(y0NvrV_KHEHyTDsm@eFOmW0{0H%=E`6y68^ z^hVmbpU*PapGwb_pO3*K?LnHxV2^b4FbebWt=(8Ik2{^FXrJ|prdW2nt)@6V=@o@# znwI@)9vmK>Uk?0L@$_6OjvO${vahw07cf4zG54lPoF)%2R>fCwq(p!S5CI}U1ggWq z+$9!7fCvzQZv^o2D4?P`T1%z!UI`6epV4oiVgVc95~xC}qqS5D9w=O;f~r)mpBP-F z!?@6Kj@DACN@rZI4D;xftH%qMtHZdE;fy;yPc>Su3F4LB)b`B^pq`Ub_T92mePJ(y9CcZHRNUmP&CJ9GB^! ObPSjXs2s=0rePvOsJ(|% zP`Ha52`+qQPbqd1M^!jtnf$}athT=;Qx7Y4H{ zW#>UY&%9u!JQqPe0gtkWQYK)JP4p;^^2xQ+TrLm0-InN_^oy2Q_ImA>I6mqZg>6~3 z^JeYsADo^KgQw#8rPLHTKvrqjXeF;;m_hZ$W?9eV$IODEXDv1CQSS3)`RR_=iGx%lJKJr()P>l!> zfqzDTHwON|gGHIM^~dt?tQC-NAY;O~5)DXTuUrD4g8N8QIkjJ)4RNl~QY(&v?W#K{ OT?7;mDu}=@5cmYR6Gp25 diff --git a/src/main/java/com/.DS_Store b/src/main/java/com/.DS_Store deleted file mode 100644 index e0512af466f9b9e96547f2cda22eb0628ed9b252..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z-NTO;8~hp~uB@kyfqk!9xfYJPIlFU_}!mO`tTTN$jCUgW&V{Ha?Ft zyPHEWXAwICv)}Ce>}Eg6{xHV4lLdXoEXJ4tipbHZ5p>sv8af$~%P}(M`}Fy>FPNaX6bs%OKu3akdPSJ2#9sZWKKE z-uhnJsh4Jn@n)=NW(NP7~;3$SN8dm064_+WLm+q1!-EBcqCyeqcDVXrGL&PRD} z8ir|o7{@24S2q*yC4YS@6n`A3mU5)Agm*A1OL=zJ%SF7LgTJgPa|ww7Vt^PR26mVM zeJB_QJInzsofsen>KVZOL4YFK8ViMT>wpHY&lqnZqJWKW2}GgM)>tTn2nbiDfU1=1 zCk9vL;1@d2)>tT1<&4XjVH`bk`FP=ScJK?C&bY0RT4I10s4~!0T?^0uYxreqANkc1 z>JbCPz&~Svx5n<+fko-F^~dt?tQDY-peUGEpaBAQ%6b|%Zb3AY~)C*$|BHp}d7^GRV`OZt_%FgbdVVcJNa1ca( zJh$Un7VzV$ta|8{+J=o9Cusn6+2q@aXvBs_#DLPtS#7lEX(Qr5V#Ntt0 b2bBVTfd-(Zu}}ye5c(02G|)f{{3-( Date: Sun, 29 Apr 2018 01:19:14 +0530 Subject: [PATCH 094/105] #44 added provision for specifying protocol --- README.md | 3 +++ .../greengerong/PreRenderSEOFilter.java | 19 +++++++++++-------- .../github/greengerong/PrerenderConfig.java | 4 ++++ .../greengerong/PrerenderSeoService.java | 8 ++++++-- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7d0af15..f2af0fc 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,9 @@ example: someproxy,someproxy1 Important for servers behind reverse proxy that need the public url to be used for pre-rendering. We usually set the original url in an http header which is added by the reverse proxy (similar to the more standard `x-forwarded-proto` and `x-forwarded-for`) +### protocol +If you specifically want to make sure that the Prerender service queries using http or https protocol, you can set the init-param `protocol` to `https` or `http` respectively. + ### Using your own prerender service If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://service.prerender.io/` diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 6eb721f..39195f9 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,20 +1,23 @@ package com.github.greengerong; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -import javax.servlet.*; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; public class PreRenderSEOFilter implements Filter { public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl"); + "blacklist", "prerenderServiceUrl", "protocol"); private PrerenderSeoService prerenderSeoService; @Override diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 90c2240..a18d314 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -79,6 +79,10 @@ public String getPrerenderToken() { public String getForwardedURLHeader() { return config.get("forwardedURLHeader"); } + + public String getProtocol() { + return config.get("protocol"); + } public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("baiduspider", diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 250b075..4ab980e 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -179,12 +179,16 @@ private void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest p } private String getRequestURL(HttpServletRequest request) { - if (prerenderConfig.getForwardedURLHeader() != null) { + if (StringUtils.isNotEmpty(prerenderConfig.getForwardedURLHeader())) { String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); - if (url != null) { + if (StringUtils.isNotEmpty(url)) { return url; } } + if (StringUtils.isNotEmpty(prerenderConfig.getProtocol())) { + String url = request.getRequestURL().toString(); + return url.replace(request.getScheme(), prerenderConfig.getProtocol()); + } return request.getRequestURL().toString(); } From 4f5c76f29e0c1fb63dcd31744bc27da1ca622e7a Mon Sep 17 00:00:00 2001 From: Karan Gujral Date: Sun, 29 Apr 2018 01:23:33 +0530 Subject: [PATCH 095/105] #44 changed readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f2af0fc..0486019 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Important for servers behind reverse proxy that need the public url to be used f We usually set the original url in an http header which is added by the reverse proxy (similar to the more standard `x-forwarded-proto` and `x-forwarded-for`) ### protocol -If you specifically want to make sure that the Prerender service queries using http or https protocol, you can set the init-param `protocol` to `https` or `http` respectively. +If you specifically want to make sure that the Prerender service queries using https or http protocol, you can set the init-param `protocol` to `https` or `http` respectively. ### Using your own prerender service From 049355625b8d074209511987a965538bbd2b082d Mon Sep 17 00:00:00 2001 From: green Date: Mon, 7 May 2018 11:11:19 +0800 Subject: [PATCH 096/105] Update .travis.yml --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index aec209e..e8f05af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: java script: "mvn clean install -U" jdk: - #- oraclejdk6 - - oraclejdk7 - - openjdk6 - - openjdk7 \ No newline at end of file + - openjdk7 + - openjdk8 From 23b6fd6370ced4c38325405c8019e26bf36c6557 Mon Sep 17 00:00:00 2001 From: Karan Gujral Date: Fri, 11 May 2018 14:04:10 -0500 Subject: [PATCH 097/105] trying to re-run CI --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0486019..edf8c2e 100644 --- a/README.md +++ b/README.md @@ -112,4 +112,4 @@ project demo test url: ## License -The MIT License (MIT) +The MIT License (MIT) \ No newline at end of file From b2583beb0a34c0566d7ddf80ce38e767ce7d6776 Mon Sep 17 00:00:00 2001 From: green Date: Wed, 1 Aug 2018 11:38:08 +0800 Subject: [PATCH 098/105] LICENSE --- LICENSE.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..43db8ab --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017-present, GreenGerong(https://github.com/greengerong), Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From d677fb0e3a99f59bd76995e596688d13c1504b3d Mon Sep 17 00:00:00 2001 From: Nathan McWilliams Date: Thu, 23 Aug 2018 17:17:37 -0400 Subject: [PATCH 099/105] change logging of final request url from log.trace to log.info --- src/main/java/com/github/greengerong/PrerenderSeoService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 4ab980e..288e869 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -331,7 +331,7 @@ private boolean beforeRender(HttpServletRequest request, HttpServletResponse res private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException { final String apiUrl = getApiUrl(getFullUrl(request)); - log.trace(String.format("Prerender proxy will send request to:%s", apiUrl)); + log.info(String.format("Prerender proxy will send request to:%s", apiUrl)); final HttpGet getMethod = getHttpGet(apiUrl); copyRequestHeaders(request, getMethod); withPrerenderToken(getMethod); From f01e29c4c0d590a380143f3b83f75e9a3702478e Mon Sep 17 00:00:00 2001 From: Todd Hooper Date: Mon, 15 Oct 2018 20:51:37 -0500 Subject: [PATCH 100/105] Add a check for our X-Prerender header Make sure requests from the prerender browser don't get sent through to Prerender.io --- .../java/com/github/greengerong/PrerenderSeoService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 4ab980e..043a4a9 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -106,6 +106,11 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI log.trace("request is for a (static) resource; intercept: no"); return false; } + + String prerenderHeader = request.getHeader("X-Prerender"); + if (StringUtils.isNotEmpty(prerenderHeader)) { + return false; + } final List whiteList = prerenderConfig.getWhitelist(); if (whiteList != null && !isInWhiteList(url, whiteList)) { From df2d5438c97ee6f55003ff3337375a27642c5a4a Mon Sep 17 00:00:00 2001 From: Todd Hooper Date: Mon, 15 Oct 2018 20:58:55 -0500 Subject: [PATCH 101/105] added a test for the X-Prerender header check --- .../greengerong/PreRenderSEOFilterTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 6776586..ba81d67 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -132,6 +132,25 @@ public void should_not_handle_when_user_agent_is_not_crawler() throws Exception verify(filterChain).doFilter(servletRequest, servletResponse); } + @Test + public void should_not_handle_when_x_prerender_header_is_present() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + when(servletRequest.getHeader("X-Prerender")).thenReturn("1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + @Test public void should_not_handle_when_url_is_a_resource() throws Exception { //given From 09a90437c5e848956842a449698cda13254c5c45 Mon Sep 17 00:00:00 2001 From: Csaba Varro Date: Wed, 24 Nov 2021 13:58:45 +0100 Subject: [PATCH 102/105] Extend user agent list --- src/main/java/com/github/greengerong/PrerenderConfig.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index a18d314..6b4e894 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -79,7 +79,7 @@ public String getPrerenderToken() { public String getForwardedURLHeader() { return config.get("forwardedURLHeader"); } - + public String getProtocol() { return config.get("protocol"); } @@ -88,7 +88,9 @@ public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview" , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot", "vkShare", - "W3C_Validator", "redditbot", "Applebot"); + "W3C_Validator", "redditbot", "Applebot", "whatsapp", "flipboard", "tumblr", "bitlybot", + "skypeuripreview", "nuzzel", "discordbot", "google page speed", "qwantify", "pinterestbot", + "bitrix link preview", "xing-contenttabreceiver", "chrome-lighthouse", "telegrambot"); final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); if (isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); From 6c415e843ce143e2eab3a21b27b00b143d68ba0b Mon Sep 17 00:00:00 2001 From: greengerong Date: Mon, 12 Jun 2023 00:38:18 +0800 Subject: [PATCH 103/105] rename word --- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 39195f9..1fa4719 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -28,9 +28,9 @@ public void init(FilterConfig filterConfig) throws ServletException { @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - boolean isPrerendered = prerenderSeoService.prerenderIfEligible( + boolean isPreRendered = prerenderSeoService.prerenderIfEligible( (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); - if (!isPrerendered) { + if (!isPreRendered) { filterChain.doFilter(servletRequest, servletResponse); } } From 17618ffac877c9e0aa755f18c6bfb1ac103da8a3 Mon Sep 17 00:00:00 2001 From: green Date: Sat, 17 Jun 2023 11:41:14 +0800 Subject: [PATCH 104/105] Update PreRenderSEOFilter.java socketTimeout --- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 1fa4719..5704025 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -17,7 +17,7 @@ public class PreRenderSEOFilter implements Filter { public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl", "protocol"); + "blacklist", "prerenderServiceUrl", "protocol","socketTimeout"); private PrerenderSeoService prerenderSeoService; @Override From 292bd36335126dd9ea3fd8329ce629efddeeb91c Mon Sep 17 00:00:00 2001 From: green Date: Sun, 9 Jul 2023 16:28:03 +0800 Subject: [PATCH 105/105] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index edf8c2e..4d3f00f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Prerender Java [![Build Status](https://travis-ci.org/greengerong/prerender-java.png)](https://travis-ci.org/greengerong/prerender-java) +Prerender Java =========================== Are you using backbone, angular, emberjs, etc, but you're unsure about the SEO implications? @@ -112,4 +112,4 @@ project demo test url: ## License -The MIT License (MIT) \ No newline at end of file +The MIT License (MIT)