From 10e9af5f840e3c14d6d3def0ba1bfb8aca816d40 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Wed, 5 Mar 2014 01:46:31 +0800 Subject: [PATCH 01/50] 1: add log for demo. 2 add html to event(due to java stream con not read more) --- demo/pom.xml | 23 +++++- demo/src/main/resources/log4j.properties | 72 +++++++++++++++++++ demo/src/main/webapp/WEB-INF/web.xml | 4 ++ prerender-java.iml | 1 + .../greengerong/PreRenderEventHandler.java | 4 +- .../greengerong/PreRenderSEOFilter.java | 29 +++++--- 6 files changed, 120 insertions(+), 13 deletions(-) create mode 100644 demo/src/main/resources/log4j.properties diff --git a/demo/pom.xml b/demo/pom.xml index 813d096..825c95c 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,11 +11,32 @@ com.github.greengerong prerender-java - 1.6.1 + 1.6.2-SNAPSHOT + + + com.github.greengerong + prerender-java-redis + 0.0.1-SNAPSHOT + + + commons-io + commons-io + 2.4 + + + org.slf4j + slf4j-log4j12 + 1.7.5 + + + log4j + log4j + 1.2.17 demo + org.mortbay.jetty diff --git a/demo/src/main/resources/log4j.properties b/demo/src/main/resources/log4j.properties new file mode 100644 index 0000000..8138149 --- /dev/null +++ b/demo/src/main/resources/log4j.properties @@ -0,0 +1,72 @@ +#log4j.rootLogger=CONSOLE,FILE +log4j.rootLogger=ALL,CONSOLE,FILE +log4j.addivity.org.apache=true + + +# 应用于控制台 +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=INFO +log4j.appender.CONSOLE.Target=System.out +log4j.appender.CONSOLE.Encoding=GBK +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +# 每天新建日志 +log4j.appender.A1=org.apache.log4j.DailyRollingFileAppender +log4j.appender.A1.File=C:/log4j/log +log4j.appender.A1.Encoding=GBK +log4j.appender.A1.Threshold=DEBUG +log4j.appender.A1.DatePattern='.'yyyy-MM-dd +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L : %m%n + +#应用于文件 +log4j.appender.FILE=org.apache.log4j.FileAppender +log4j.appender.FILE.File=C:/log4j/file.log +log4j.appender.FILE.Append=false +log4j.appender.FILE.Encoding=GBK +log4j.appender.FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +# 应用于文件回滚 +log4j.appender.ROLLING_FILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLING_FILE.Threshold=ERROR +log4j.appender.ROLLING_FILE.File=rolling.log +log4j.appender.ROLLING_FILE.Append=true +log4j.appender.CONSOLE_FILE.Encoding=GBK +log4j.appender.ROLLING_FILE.MaxFileSize=10KB +log4j.appender.ROLLING_FILE.MaxBackupIndex=1 +log4j.appender.ROLLING_FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLING_FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +#自定义Appender +log4j.appender.im = net.cybercorlin.util.logger.appender.IMAppender +log4j.appender.im.host = mail.cybercorlin.net +log4j.appender.im.username = username +log4j.appender.im.password = password +log4j.appender.im.recipient = yyflyons@163.com +log4j.appender.im.layout=org.apache.log4j.PatternLayout +log4j.appender.im.layout.ConversionPattern =[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +#应用于socket +log4j.appender.SOCKET=org.apache.log4j.RollingFileAppender +log4j.appender.SOCKET.RemoteHost=localhost +log4j.appender.SOCKET.Port=5001 +log4j.appender.SOCKET.LocationInfo=true +# Set up for Log Facter 5 +log4j.appender.SOCKET.layout=org.apache.log4j.PatternLayout +log4j.appender.SOCET.layout.ConversionPattern=[start]%d{DATE}[DATE]%n%p[PRIORITY]%n%x[NDC]%n%t[THREAD]%n%c[CATEGORY]%n%m[MESSAGE]%n%n +# Log Factor 5 Appender +log4j.appender.LF5_APPENDER=org.apache.log4j.lf5.LF5Appender +log4j.appender.LF5_APPENDER.MaxNumberOfRecords=2000 + +# 发送日志给邮件 +log4j.appender.MAIL=org.apache.log4j.net.SMTPAppender +log4j.appender.MAIL.Threshold=FATAL +log4j.appender.MAIL.BufferSize=10 +log4j.appender.MAIL.From=yyflyons@163.com +log4j.appender.MAIL.SMTPHost=www.wusetu.com +log4j.appender.MAIL.Subject=Log4J Message +log4j.appender.MAIL.To=yyflyons@126.com +log4j.appender.MAIL.layout=org.apache.log4j.PatternLayout +log4j.appender.MAIL.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n diff --git a/demo/src/main/webapp/WEB-INF/web.xml b/demo/src/main/webapp/WEB-INF/web.xml index eadd7d5..c5e037b 100644 --- a/demo/src/main/webapp/WEB-INF/web.xml +++ b/demo/src/main/webapp/WEB-INF/web.xml @@ -15,6 +15,10 @@ crawlerUserAgents YahooSeeker + + preRenderEventHandler + com.github.greengerong.PrerenderRedisCache + prerender diff --git a/prerender-java.iml b/prerender-java.iml index cbd3c69..7b5c7a6 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -6,6 +6,7 @@ + diff --git a/src/main/java/com/github/greengerong/PreRenderEventHandler.java b/src/main/java/com/github/greengerong/PreRenderEventHandler.java index 9e0d74e..9669afb 100644 --- a/src/main/java/com/github/greengerong/PreRenderEventHandler.java +++ b/src/main/java/com/github/greengerong/PreRenderEventHandler.java @@ -9,5 +9,7 @@ public interface PreRenderEventHandler { String beforeRender(HttpServletRequest clientRequest); - void afterRender(HttpServletRequest clientRequest, HttpResponse prerenderResponse); + void afterRender(HttpServletRequest clientRequest, HttpResponse prerenderResponse, String html); + + void destroy(); } diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 4350fa6..8914574 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -10,6 +10,7 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.message.BasicHeader; import org.apache.http.message.HeaderGroup; +import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,7 +19,6 @@ import javax.servlet.http.HttpServletResponse; import java.io.Closeable; import java.io.IOException; -import java.io.OutputStream; import java.io.PrintWriter; import java.net.URI; import java.net.URISyntaxException; @@ -40,7 +40,6 @@ public class PreRenderSEOFilter implements Filter { public void init(FilterConfig filterConfig) throws ServletException { this.prerenderConfig = new PrerenderConfig(filterConfig); this.httpClient = getHttpClient(); - this.preRenderEventHandler = prerenderConfig.getEventHandler(); } protected CloseableHttpClient getHttpClient() { @@ -54,7 +53,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo final HttpServletRequest request = (HttpServletRequest) servletRequest; final HttpServletResponse response = (HttpServletResponse) servletResponse; if (shouldShowPrerenderedPage(request)) { - + this.preRenderEventHandler = prerenderConfig.getEventHandler(); if (beforeRender(request, response) || proxyPrerenderedPageResponse(request, response)) { return; } @@ -90,9 +89,9 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer try { proxyResponse = httpClient.execute(getMethod); if (proxyResponse.getStatusLine().getStatusCode() == HTTP_OK) { - afterRender(request, proxyResponse); copyResponseHeaders(proxyResponse, response); - copyResponseEntity(proxyResponse, response); + final String html = copyResponseEntity(proxyResponse, response); + afterRender(request, proxyResponse, html); return true; } } finally { @@ -105,9 +104,9 @@ protected HttpGet getHttpGet(String apiUrl) { return new HttpGet(apiUrl); } - private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse) { + private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse, String html) { if (preRenderEventHandler != null) { - preRenderEventHandler.afterRender(request, proxyResponse); + preRenderEventHandler.afterRender(request, proxyResponse, html); } } @@ -133,16 +132,20 @@ protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletRespon /** * Copy response body data (the entity) from the proxy to the servlet client. */ - protected void copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) throws IOException { + protected String copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) throws IOException { HttpEntity entity = proxyResponse.getEntity(); if (entity != null) { - OutputStream servletOutputStream = servletResponse.getOutputStream(); + PrintWriter printWriter = servletResponse.getWriter(); try { - entity.writeTo(servletOutputStream); + final String html = EntityUtils.toString(entity); + printWriter.write(html); + printWriter.flush(); + return html; } finally { - closeQuietly(servletOutputStream); + closeQuietly(printWriter); } } + return ""; } protected void closeQuietly(Closeable closeable) { @@ -217,6 +220,10 @@ private String getFullUrl(HttpServletRequest request) { @Override public void destroy() { prerenderConfig = null; + if (preRenderEventHandler != null) { + preRenderEventHandler.destroy(); + preRenderEventHandler = null; + } closeQuietly(httpClient); } From 1237fc128f1cba6e7b60af43d65dd12d73f2a98f Mon Sep 17 00:00:00 2001 From: greengerongg Date: Wed, 5 Mar 2014 01:54:39 +0800 Subject: [PATCH 02/50] [maven-release-plugin] prepare release prerender-java-1.6.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 47e46f1..fdbd6a4 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.2-SNAPSHOT + 1.6.2 jar prerender-java From c6ccb4a51b096bf41ef81c67ade43b49927793b7 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Wed, 5 Mar 2014 01:54:52 +0800 Subject: [PATCH 03/50] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fdbd6a4..b2eaf60 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.2 + 1.6.3-SNAPSHOT jar prerender-java From 102225191152fbed3b2172e0a3da557d07f716d1 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Wed, 5 Mar 2014 01:58:14 +0800 Subject: [PATCH 04/50] release for cache event 1.6.2 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ae6af0..1eb1c85 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.6.1 + 1.6.2 ## How it works From 3cd62fd1f06fe4589e7b607ebe32b5219c543b28 Mon Sep 17 00:00:00 2001 From: greengerongg Date: Wed, 5 Mar 2014 09:52:28 +0800 Subject: [PATCH 05/50] change demo --- .gitignore | 1 + demo/pom.xml | 2 +- demo/src/main/resources/log4j.properties | 15 +++------------ prerender-java.iml | 2 +- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index a9aaddb..71d7773 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ target/ **/target/ **/.idea/ +*.iml diff --git a/demo/pom.xml b/demo/pom.xml index 825c95c..2552382 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,7 +11,7 @@ com.github.greengerong prerender-java - 1.6.2-SNAPSHOT + 1.6.2 com.github.greengerong diff --git a/demo/src/main/resources/log4j.properties b/demo/src/main/resources/log4j.properties index 8138149..3074ab0 100644 --- a/demo/src/main/resources/log4j.properties +++ b/demo/src/main/resources/log4j.properties @@ -1,5 +1,5 @@ #log4j.rootLogger=CONSOLE,FILE -log4j.rootLogger=ALL,CONSOLE,FILE +log4j.rootLogger=DEBUG,CONSOLE log4j.addivity.org.apache=true @@ -39,15 +39,6 @@ log4j.appender.ROLLING_FILE.MaxBackupIndex=1 log4j.appender.ROLLING_FILE.layout=org.apache.log4j.PatternLayout log4j.appender.ROLLING_FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n -#自定义Appender -log4j.appender.im = net.cybercorlin.util.logger.appender.IMAppender -log4j.appender.im.host = mail.cybercorlin.net -log4j.appender.im.username = username -log4j.appender.im.password = password -log4j.appender.im.recipient = yyflyons@163.com -log4j.appender.im.layout=org.apache.log4j.PatternLayout -log4j.appender.im.layout.ConversionPattern =[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n - #应用于socket log4j.appender.SOCKET=org.apache.log4j.RollingFileAppender log4j.appender.SOCKET.RemoteHost=localhost @@ -64,9 +55,9 @@ log4j.appender.LF5_APPENDER.MaxNumberOfRecords=2000 log4j.appender.MAIL=org.apache.log4j.net.SMTPAppender log4j.appender.MAIL.Threshold=FATAL log4j.appender.MAIL.BufferSize=10 -log4j.appender.MAIL.From=yyflyons@163.com +log4j.appender.MAIL.From= log4j.appender.MAIL.SMTPHost=www.wusetu.com log4j.appender.MAIL.Subject=Log4J Message -log4j.appender.MAIL.To=yyflyons@126.com +log4j.appender.MAIL.To= log4j.appender.MAIL.layout=org.apache.log4j.PatternLayout log4j.appender.MAIL.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n diff --git a/prerender-java.iml b/prerender-java.iml index 7b5c7a6..968be0a 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -4,9 +4,9 @@ + - From 9dd185200f2d7517751b3758f798924b513bb9e9 Mon Sep 17 00:00:00 2001 From: Shahar Talmi Date: Fri, 11 Apr 2014 01:11:35 +0300 Subject: [PATCH 06/50] add init parameter for forwarded url header important for servers behind reverse proxy that need the public url to be used for pre-rendering. we usually set the original url in an http header which is added by the reverse proxy (similar to the more standard x-forwarded-proto and x-forwarded-for) --- README.md | 3 ++ .../greengerong/PreRenderSEOFilter.java | 19 ++++++++--- .../github/greengerong/PrerenderConfig.java | 4 +++ .../greengerong/PreRenderSEOFilterTest.java | 33 +++++++++++++++++++ 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1eb1c85..6c67fc5 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,9 @@ example: someproxy,someproxy1 ### blacklist +### forwardedURLHeader +Important for servers behind reverse proxy that need the public url to be used for pre-rendering. +We usually set the original url in an http header which is added by the reverse proxy (similar to the more standard `x-forwarded-proto` and `x-forwarded-for`) ### Using your own prerender service diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 8914574..f4cb717 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -207,14 +207,23 @@ protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest } } + private String getRequestURL(HttpServletRequest request) { + if (prerenderConfig.getForwardedURLHeader() != null) { + String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); + if (url != null) { + return url; + } + } + return request.getRequestURL().toString(); + } + private String getFullUrl(HttpServletRequest request) { - final StringBuffer url = request.getRequestURL(); + final String url = getRequestURL(request); final String queryString = request.getQueryString(); if (queryString != null) { - url.append('?'); - url.append(queryString); + return url + '?' + queryString; } - return url.toString(); + return url; } @Override @@ -229,7 +238,7 @@ public void destroy() { private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { final String userAgent = request.getHeader("User-Agent"); - final String url = request.getRequestURL().toString(); + final String url = getRequestURL(request); final String referer = request.getHeader("Referer"); log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index c5daa05..38b810e 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -53,6 +53,10 @@ public String getPrerenderToken() { return filterConfig.getInitParameter("prerenderToken"); } + public String getForwardedURLHeader() { + return filterConfig.getInitParameter("forwardedURLHeader"); + } + public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly"); diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 1ac7521..fe35749 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -266,4 +266,37 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and verify(httpClient).execute(httpGet); verify(filterChain, never()).doFilter(servletRequest, servletResponse); } + + @Test + public void should_use_request_url_from_custom_header_if_available() throws Exception { + //given + when(filterConfig.getInitParameter("forwardedURLHeader")).thenReturn("X-Forwarded-URL"); + when(filterConfig.getInitParameter("whitelist")).thenReturn("http://my.public.domain.com/"); + when(filterConfig.getInitParameter("blacklist")).thenReturn("http://localhost/test"); + + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getHeader("X-Forwarded-URL")).thenReturn("http://my.public.domain.com/"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put("_escaped_fragment_", ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } } From 61ae780c5557fe814c97428c1d9c6740083680ae Mon Sep 17 00:00:00 2001 From: green Date: Tue, 15 Apr 2014 22:40:25 +1000 Subject: [PATCH 07/50] java doc update for local java 8 compile --- pom.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b2eaf60..bb62f67 100644 --- a/pom.xml +++ b/pom.xml @@ -114,8 +114,13 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.7 + 2.8.1 + + http://docs.oracel.com/javase/6/docs/api/ + http://docs.oracel.com/javaee/6/api/ + + ${java.home}/../bin/javadoc From 44a64fb0c0461d2461cf6f8e0974447e61e86e3d Mon Sep 17 00:00:00 2001 From: greengerongg Date: Tue, 15 Apr 2014 22:51:48 +1000 Subject: [PATCH 08/50] change back java doc --- pom.xml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index bb62f67..380a7a8 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,5 @@ - + 4.0.0 com.github.greengerong @@ -26,7 +27,7 @@ oss-parent 7 - + UTF-8 4.11 @@ -115,13 +116,6 @@ org.apache.maven.plugins maven-javadoc-plugin 2.8.1 - - - http://docs.oracel.com/javase/6/docs/api/ - http://docs.oracel.com/javaee/6/api/ - - ${java.home}/../bin/javadoc - attach-javadoc From 1b5f9ef98a2be2df63bfc808c2267613a0ac13f1 Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Wed, 28 May 2014 21:38:18 +0300 Subject: [PATCH 09/50] refactoring prerender service out of filter --- .../greengerong/PreRenderSEOFilter.java | 353 ++++++++++-------- .../greengerong/PreRenderSEOFilterTest.java | 28 +- 2 files changed, 208 insertions(+), 173 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index f4cb717..89270de 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -29,39 +29,200 @@ import static com.google.common.collect.FluentIterable.from; public class PreRenderSEOFilter implements Filter { + private PrerenderService prerenderService; - private final static Logger log = LoggerFactory.getLogger(PreRenderSEOFilter.class); + @Override + public void init(FilterConfig filterConfig) throws ServletException { + this.prerenderService = new PrerenderService(new PrerenderConfig(filterConfig)); + } + + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) + throws IOException, ServletException { + boolean isPrerendered = prerenderService.tryPrerender( + (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); + if (!isPrerendered) { + filterChain.doFilter(servletRequest, servletResponse); + } + } + + @Override + public void destroy() { + prerenderService.destroy(); + } + + protected void setPrerenderService(PrerenderService prerenderService) { + this.prerenderService = prerenderService; + } +} + +class PrerenderService { + + public static final int HTTP_OK = 200; private CloseableHttpClient httpClient; - private PreRenderEventHandler preRenderEventHandler; private PrerenderConfig prerenderConfig; - public static final int HTTP_OK = 200; + private PreRenderEventHandler preRenderEventHandler; - @Override - public void init(FilterConfig filterConfig) throws ServletException { - this.prerenderConfig = new PrerenderConfig(filterConfig); + /** + * These are the "hop-by-hop" headers that should not be copied. + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html + * I use an HttpClient HeaderGroup class instead of Set because this + * approach does case insensitive lookup faster. + */ + protected static final HeaderGroup hopByHopHeaders; + + public PrerenderService(PrerenderConfig prerenderConfig) { + this.prerenderConfig = prerenderConfig; this.httpClient = getHttpClient(); } - protected CloseableHttpClient getHttpClient() { - return prerenderConfig.getHttpClient(); + static { + hopByHopHeaders = new HeaderGroup(); + String[] headers = new String[]{ + "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", + "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; + for (String header : headers) { + hopByHopHeaders.addHeader(new BasicHeader(header, null)); + } } - @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) - throws IOException, ServletException { + public void destroy() { + if (preRenderEventHandler != null) { + preRenderEventHandler.destroy(); + } + closeQuietly(httpClient); + } + + public boolean tryPrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) { try { - final HttpServletRequest request = (HttpServletRequest) servletRequest; - final HttpServletResponse response = (HttpServletResponse) servletResponse; - if (shouldShowPrerenderedPage(request)) { - this.preRenderEventHandler = prerenderConfig.getEventHandler(); - if (beforeRender(request, response) || proxyPrerenderedPageResponse(request, response)) { - return; - } - } + if (handlePrerender(servletRequest, servletResponse)) + return true; } catch (Exception e) { log.error("Prerender service error", e); } - filterChain.doFilter(servletRequest, servletResponse); + return false; + } + + private boolean handlePrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) + throws URISyntaxException, IOException { + if (shouldShowPrerenderedPage(servletRequest)) { + this.preRenderEventHandler = prerenderConfig.getEventHandler(); + if (beforeRender(servletRequest, servletResponse) || proxyPrerenderedPageResponse(servletRequest, servletResponse)) { + return true; + } + } + return false; + } + + private boolean hasEscapedFragment(HttpServletRequest request) { + return request.getParameterMap().containsKey("_escaped_fragment_"); + } + + public String getApiUrl(String url) { + String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); + if (!prerenderServiceUrl.endsWith("/")) { + prerenderServiceUrl += "/"; + } + return prerenderServiceUrl + url; + } + + private boolean isInBlackList(final String url, final String referer, List blacklist) { + return from(blacklist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + final Pattern pattern = Pattern.compile(regex); + return pattern.matcher(url).matches() || + (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); + } + }); + } + + private boolean isInSearchUserAgent(final String userAgent) { + return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return userAgent.toLowerCase().indexOf(item.toLowerCase()) >= 0; + } + }); + } + + private final static Logger log = LoggerFactory.getLogger(PrerenderService.class); + + public boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { + final String userAgent = request.getHeader("User-Agent"); + final String url = getRequestURL(request); + final String referer = request.getHeader("Referer"); + + log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); + + if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { + log.trace("Request is not HTTP GET; intercept: no"); + return false; + } + + if (isInResources(url)) { + log.trace("request is for a (static) resource; intercept: no"); + return false; + } + + final List whiteList = prerenderConfig.getWhitelist(); + if (whiteList != null && !isInWhiteList(url, whiteList)) { + log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); + return false; + } + + final List blacklist = prerenderConfig.getBlacklist(); + if (blacklist != null && isInBlackList(url, referer, blacklist)) { + log.trace("Blacklist is enabled, and this request is listed; intercept: no"); + return false; + } + + if (hasEscapedFragment(request)) { + log.trace("Request Has _escaped_fragment_; intercept: yes"); + return true; + } + + if (StringUtils.isBlank(userAgent)) { + log.trace("Request has blank userAgent; intercept: no"); + return false; + } + + if (!isInSearchUserAgent(userAgent)) { + log.trace("Request User-Agent is not a search bot; intercept: no"); + return false; + } + + log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); + return true; + } + + + public String getRequestURL(HttpServletRequest request) { + if (prerenderConfig.getForwardedURLHeader() != null) { + String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); + if (url != null) { + return url; + } + } + return request.getRequestURL().toString(); + } + + private boolean isInResources(final String url) { + return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return url.contains(item.toLowerCase()); + } + }); + } + + private boolean isInWhiteList(final String url, List whitelist) { + return from(whitelist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + return Pattern.compile(regex).matcher(url).matches(); + } + }); } private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) throws IOException { @@ -100,10 +261,6 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer return false; } - protected HttpGet getHttpGet(String apiUrl) { - return new HttpGet(apiUrl); - } - private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse, String html) { if (preRenderEventHandler != null) { preRenderEventHandler.afterRender(request, proxyResponse, html); @@ -158,22 +315,13 @@ protected void closeQuietly(Closeable closeable) { } } - /** - * These are the "hop-by-hop" headers that should not be copied. - * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html - * I use an HttpClient HeaderGroup class instead of Set because this - * approach does case insensitive lookup faster. - */ - protected static final HeaderGroup hopByHopHeaders; - - static { - hopByHopHeaders = new HeaderGroup(); - String[] headers = new String[]{ - "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", - "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; - for (String header : headers) { - hopByHopHeaders.addHeader(new BasicHeader(header, null)); + private String getFullUrl(HttpServletRequest request) { + final String url = getRequestURL(request); + final String queryString = request.getQueryString(); + if (queryString != null) { + return url + '?' + queryString; } + return url; } /** @@ -207,131 +355,12 @@ protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest } } - private String getRequestURL(HttpServletRequest request) { - if (prerenderConfig.getForwardedURLHeader() != null) { - String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); - if (url != null) { - return url; - } - } - return request.getRequestURL().toString(); - } - - private String getFullUrl(HttpServletRequest request) { - final String url = getRequestURL(request); - final String queryString = request.getQueryString(); - if (queryString != null) { - return url + '?' + queryString; - } - return url; - } - - @Override - public void destroy() { - prerenderConfig = null; - if (preRenderEventHandler != null) { - preRenderEventHandler.destroy(); - preRenderEventHandler = null; - } - closeQuietly(httpClient); - } - - private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { - final String userAgent = request.getHeader("User-Agent"); - final String url = getRequestURL(request); - final String referer = request.getHeader("Referer"); - - log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); - - if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { - log.trace("Request is not HTTP GET; intercept: no"); - return false; - } - - if (isInResources(url)) { - log.trace("request is for a (static) resource; intercept: no"); - return false; - } - - final List whiteList = prerenderConfig.getWhitelist(); - if (whiteList != null && !isInWhiteList(url, whiteList)) { - log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); - return false; - } - - final List blacklist = prerenderConfig.getBlacklist(); - if (blacklist != null && isInBlackList(url, referer, blacklist)) { - log.trace("Blacklist is enabled, and this request is listed; intercept: no"); - return false; - } - - if (hasEscapedFragment(request)) { - log.trace("Request Has _escaped_fragment_; intercept: yes"); - return true; - } - - if (StringUtils.isBlank(userAgent)) { - log.trace("Request has blank userAgent; intercept: no"); - return false; - } - - if (!isInSearchUserAgent(userAgent)) { - log.trace("Request User-Agent is not a search bot; intercept: no"); - return false; - } - - log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); - return true; - } - - private boolean hasEscapedFragment(HttpServletRequest request) { - return request.getParameterMap().containsKey("_escaped_fragment_"); - } - - private String getApiUrl(String url) { - String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); - if (!prerenderServiceUrl.endsWith("/")) { - prerenderServiceUrl += "/"; - } - return prerenderServiceUrl + url; - } - private boolean isInBlackList(final String url, final String referer, List blacklist) { - return from(blacklist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - final Pattern pattern = Pattern.compile(regex); - return pattern.matcher(url).matches() || - (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); - } - }); - } - - private boolean isInWhiteList(final String url, List whitelist) { - return from(whitelist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - return Pattern.compile(regex).matcher(url).matches(); - } - }); - } - - private boolean isInResources(final String url) { - return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return url.contains(item.toLowerCase()); - } - }); + protected HttpGet getHttpGet(String apiUrl) { + return new HttpGet(apiUrl); } - private boolean isInSearchUserAgent(final String userAgent) { - return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return userAgent.toLowerCase().indexOf(item.toLowerCase()) >= 0; - } - }); + protected CloseableHttpClient getHttpClient() { + return prerenderConfig.getHttpClient(); } - } diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index fe35749..d33edff 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -15,6 +15,7 @@ import javax.servlet.FilterChain; import javax.servlet.FilterConfig; +import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.util.Enumeration; @@ -50,13 +51,18 @@ public class PreRenderSEOFilterTest { public void setUp() throws Exception { preRenderSEOFilter = new PreRenderSEOFilter() { @Override - protected CloseableHttpClient getHttpClient() { - return httpClient; - } - - @Override - protected HttpGet getHttpGet(String apiUrl) { - return httpGet; + public void init(FilterConfig filterConfig) throws ServletException { + setPrerenderService(new PrerenderService(new PrerenderConfig(filterConfig)) { + @Override + protected CloseableHttpClient getHttpClient() { + return httpClient; + } + + @Override + protected HttpGet getHttpGet(String apiUrl) { + return httpGet; + } + }); } }; } @@ -91,7 +97,7 @@ public void should_handle_when_url_with_escaped_fragment_() throws Exception { final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -193,7 +199,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when @@ -256,7 +262,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when @@ -289,7 +295,7 @@ public void should_use_request_url_from_custom_header_if_available() throws Exce final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when From b25cc9f8d87b1971bd327e8bdf146f4538079faa Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Wed, 28 May 2014 22:15:03 +0300 Subject: [PATCH 10/50] removed prerender config dependency on filter configuration --- .../greengerong/PreRenderSEOFilter.java | 351 +----------------- .../github/greengerong/PrerenderConfig.java | 28 +- .../greengerong/PrerenderSeoService.java | 339 +++++++++++++++++ .../greengerong/PreRenderSEOFilterTest.java | 10 +- 4 files changed, 376 insertions(+), 352 deletions(-) create mode 100644 src/main/java/com/github/greengerong/PrerenderSeoService.java diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 89270de..21963b1 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,45 +1,30 @@ package com.github.greengerong; - -import com.google.common.base.Predicate; -import org.apache.commons.lang.StringUtils; -import org.apache.http.*; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIUtils; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.message.BasicHeader; -import org.apache.http.message.HeaderGroup; -import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.google.common.collect.Lists; import javax.servlet.*; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import java.io.Closeable; import java.io.IOException; -import java.io.PrintWriter; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Enumeration; +import java.util.HashMap; import java.util.List; -import java.util.regex.Pattern; - -import static com.google.common.collect.FluentIterable.from; +import java.util.Map; public class PreRenderSEOFilter implements Filter { - private PrerenderService prerenderService; + public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", + "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", + "blacklist", "prerenderServiceUrl"); + private PrerenderSeoService prerenderSeoService; @Override public void init(FilterConfig filterConfig) throws ServletException { - this.prerenderService = new PrerenderService(new PrerenderConfig(filterConfig)); + this.prerenderSeoService = new PrerenderSeoService(new PrerenderConfig(toMap(filterConfig))); } @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - boolean isPrerendered = prerenderService.tryPrerender( + boolean isPrerendered = prerenderSeoService.tryPrerender( (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); if (!isPrerendered) { filterChain.doFilter(servletRequest, servletResponse); @@ -48,319 +33,19 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo @Override public void destroy() { - prerenderService.destroy(); - } - - protected void setPrerenderService(PrerenderService prerenderService) { - this.prerenderService = prerenderService; - } -} - -class PrerenderService { - - public static final int HTTP_OK = 200; - private CloseableHttpClient httpClient; - private PrerenderConfig prerenderConfig; - private PreRenderEventHandler preRenderEventHandler; - - /** - * These are the "hop-by-hop" headers that should not be copied. - * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html - * I use an HttpClient HeaderGroup class instead of Set because this - * approach does case insensitive lookup faster. - */ - protected static final HeaderGroup hopByHopHeaders; - - public PrerenderService(PrerenderConfig prerenderConfig) { - this.prerenderConfig = prerenderConfig; - this.httpClient = getHttpClient(); - } - - static { - hopByHopHeaders = new HeaderGroup(); - String[] headers = new String[]{ - "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", - "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; - for (String header : headers) { - hopByHopHeaders.addHeader(new BasicHeader(header, null)); - } - } - - public void destroy() { - if (preRenderEventHandler != null) { - preRenderEventHandler.destroy(); - } - closeQuietly(httpClient); - } - - public boolean tryPrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) { - try { - if (handlePrerender(servletRequest, servletResponse)) - return true; - } catch (Exception e) { - log.error("Prerender service error", e); - } - return false; - } - - private boolean handlePrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) - throws URISyntaxException, IOException { - if (shouldShowPrerenderedPage(servletRequest)) { - this.preRenderEventHandler = prerenderConfig.getEventHandler(); - if (beforeRender(servletRequest, servletResponse) || proxyPrerenderedPageResponse(servletRequest, servletResponse)) { - return true; - } - } - return false; - } - - private boolean hasEscapedFragment(HttpServletRequest request) { - return request.getParameterMap().containsKey("_escaped_fragment_"); - } - - public String getApiUrl(String url) { - String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); - if (!prerenderServiceUrl.endsWith("/")) { - prerenderServiceUrl += "/"; - } - return prerenderServiceUrl + url; - } - - private boolean isInBlackList(final String url, final String referer, List blacklist) { - return from(blacklist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - final Pattern pattern = Pattern.compile(regex); - return pattern.matcher(url).matches() || - (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); - } - }); - } - - private boolean isInSearchUserAgent(final String userAgent) { - return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return userAgent.toLowerCase().indexOf(item.toLowerCase()) >= 0; - } - }); - } - - private final static Logger log = LoggerFactory.getLogger(PrerenderService.class); - - public boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { - final String userAgent = request.getHeader("User-Agent"); - final String url = getRequestURL(request); - final String referer = request.getHeader("Referer"); - - log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); - - if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { - log.trace("Request is not HTTP GET; intercept: no"); - return false; - } - - if (isInResources(url)) { - log.trace("request is for a (static) resource; intercept: no"); - return false; - } - - final List whiteList = prerenderConfig.getWhitelist(); - if (whiteList != null && !isInWhiteList(url, whiteList)) { - log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); - return false; - } - - final List blacklist = prerenderConfig.getBlacklist(); - if (blacklist != null && isInBlackList(url, referer, blacklist)) { - log.trace("Blacklist is enabled, and this request is listed; intercept: no"); - return false; - } - - if (hasEscapedFragment(request)) { - log.trace("Request Has _escaped_fragment_; intercept: yes"); - return true; - } - - if (StringUtils.isBlank(userAgent)) { - log.trace("Request has blank userAgent; intercept: no"); - return false; - } - - if (!isInSearchUserAgent(userAgent)) { - log.trace("Request User-Agent is not a search bot; intercept: no"); - return false; - } - - log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); - return true; - } - - - public String getRequestURL(HttpServletRequest request) { - if (prerenderConfig.getForwardedURLHeader() != null) { - String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); - if (url != null) { - return url; - } - } - return request.getRequestURL().toString(); - } - - private boolean isInResources(final String url) { - return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return url.contains(item.toLowerCase()); - } - }); - } - - private boolean isInWhiteList(final String url, List whitelist) { - return from(whitelist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - return Pattern.compile(regex).matcher(url).matches(); - } - }); - } - - private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) throws IOException { - if (preRenderEventHandler != null) { - final String html = preRenderEventHandler.beforeRender(request); - if (StringUtils.isNotBlank(html)) { - final PrintWriter writer = response.getWriter(); - writer.write(html); - writer.flush(); - closeQuietly(writer); - return true; - } - } - return false; - } - - private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException { - final String apiUrl = getApiUrl(getFullUrl(request)); - log.trace(String.format("Prerender proxy will send request to:%s", apiUrl)); - final HttpGet getMethod = getHttpGet(apiUrl); - copyRequestHeaders(request, getMethod); - withPrerenderToken(getMethod); - CloseableHttpResponse proxyResponse = null; - - try { - proxyResponse = httpClient.execute(getMethod); - if (proxyResponse.getStatusLine().getStatusCode() == HTTP_OK) { - copyResponseHeaders(proxyResponse, response); - final String html = copyResponseEntity(proxyResponse, response); - afterRender(request, proxyResponse, html); - return true; - } - } finally { - closeQuietly(proxyResponse); - } - return false; + prerenderSeoService.destroy(); } - private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse, String html) { - if (preRenderEventHandler != null) { - preRenderEventHandler.afterRender(request, proxyResponse, html); - } - } - - private void withPrerenderToken(HttpRequest proxyRequest) { - final String token = prerenderConfig.getPrerenderToken(); - //for new version prerender with token. - if (StringUtils.isNotBlank(token)) { - proxyRequest.addHeader("X-Prerender-Token", token); - } - } - - /** - * Copy proxied response headers back to the servlet client. - */ - protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { - for (Header header : proxyResponse.getAllHeaders()) { - if (!hopByHopHeaders.containsHeader(header.getName())) { - servletResponse.addHeader(header.getName(), header.getValue()); - } - } - } - - /** - * Copy response body data (the entity) from the proxy to the servlet client. - */ - protected String copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) throws IOException { - HttpEntity entity = proxyResponse.getEntity(); - if (entity != null) { - PrintWriter printWriter = servletResponse.getWriter(); - try { - final String html = EntityUtils.toString(entity); - printWriter.write(html); - printWriter.flush(); - return html; - } finally { - closeQuietly(printWriter); - } - } - return ""; - } - - protected void closeQuietly(Closeable closeable) { - try { - if (closeable != null) { - closeable.close(); - } - } catch (IOException e) { - log.error("Close proxy error", e); - } - } - - private String getFullUrl(HttpServletRequest request) { - final String url = getRequestURL(request); - final String queryString = request.getQueryString(); - if (queryString != null) { - return url + '?' + queryString; - } - return url; + protected void setPrerenderSeoService(PrerenderSeoService prerenderSeoService) { + this.prerenderSeoService = prerenderSeoService; } - /** - * Copy request headers from the servlet client to the proxy request. - * - * @throws URISyntaxException - */ - protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) throws URISyntaxException { - // Get an Enumeration of all of the header names sent by the client - Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); - while (enumerationOfHeaderNames.hasMoreElements()) { - String headerName = (String) enumerationOfHeaderNames.nextElement(); - //Instead the content-length is effectively set via InputStreamEntity - if (!headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { - Enumeration headers = servletRequest.getHeaders(headerName); - while (headers.hasMoreElements()) {//sometimes more than one value - String headerValue = (String) headers.nextElement(); - // In case the proxy host is running multiple virtual servers, - // rewrite the Host header to ensure that we get content from - // the correct virtual server - if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { - HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); - headerValue = host.getHostName(); - if (host.getPort() != -1) { - headerValue += ":" + host.getPort(); - } - } - proxyRequest.addHeader(headerName, headerValue); - } - } + protected Map toMap(FilterConfig filterConfig) { + HashMap config = new HashMap(); + for (String parameterName : PARAMETER_NAMES) { + config.put(parameterName, filterConfig.getInitParameter(parameterName)); } - } - - - protected HttpGet getHttpGet(String apiUrl) { - return new HttpGet(apiUrl); - } - - protected CloseableHttpClient getHttpClient() { - return prerenderConfig.getHttpClient(); + return config; } } + diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 38b810e..3407848 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -12,20 +12,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.FilterConfig; import java.util.Arrays; import java.util.List; +import java.util.Map; public class PrerenderConfig { private final static Logger log = LoggerFactory.getLogger(PrerenderConfig.class); - private FilterConfig filterConfig; + private Map config; - public PrerenderConfig(FilterConfig filterConfig) { - this.filterConfig = filterConfig; + public PrerenderConfig(Map config) { + this.config = config; } public PreRenderEventHandler getEventHandler() { - final String preRenderEventHandler = filterConfig.getInitParameter("preRenderEventHandler"); + final String preRenderEventHandler = config.get("preRenderEventHandler"); if (StringUtils.isNotBlank(preRenderEventHandler)) { try { return (PreRenderEventHandler) Class.forName(preRenderEventHandler).newInstance(); @@ -38,9 +38,9 @@ public PreRenderEventHandler getEventHandler() { public CloseableHttpClient getHttpClient() { HttpClientBuilder builder = HttpClients.custom(); - final String proxy = filterConfig.getInitParameter("proxy"); + final String proxy = config.get("proxy"); if (StringUtils.isNotBlank(proxy)) { - final int proxyPort = Integer.parseInt(filterConfig.getInitParameter("proxyPort")); + final int proxyPort = Integer.parseInt(config.get("proxyPort")); DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); builder = builder.setRoutePlanner(routePlanner); } @@ -50,17 +50,17 @@ public CloseableHttpClient getHttpClient() { } public String getPrerenderToken() { - return filterConfig.getInitParameter("prerenderToken"); + return config.get("prerenderToken"); } public String getForwardedURLHeader() { - return filterConfig.getInitParameter("forwardedURLHeader"); + return config.get("forwardedURLHeader"); } public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly"); - final String crawlerUserAgentsFromConfig = filterConfig.getInitParameter("crawlerUserAgents"); + final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); if (StringUtils.isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); } @@ -73,7 +73,7 @@ public List getExtensionsToIgnore() { ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent"); - final String extensionsToIgnoreFromConfig = filterConfig.getInitParameter("extensionsToIgnore"); + final String extensionsToIgnoreFromConfig = config.get("extensionsToIgnore"); if (StringUtils.isNotBlank(extensionsToIgnoreFromConfig)) { extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); } @@ -82,7 +82,7 @@ public List getExtensionsToIgnore() { } public List getWhitelist() { - final String whitelist = filterConfig.getInitParameter("whitelist"); + final String whitelist = config.get("whitelist"); if (StringUtils.isNotBlank(whitelist)) { return Arrays.asList(whitelist.trim().split(",")); } @@ -90,7 +90,7 @@ public List getWhitelist() { } public List getBlacklist() { - final String blacklist = filterConfig.getInitParameter("blacklist"); + final String blacklist = config.get("blacklist"); if (StringUtils.isNotBlank(blacklist)) { return Arrays.asList(blacklist.trim().split(",")); } @@ -98,7 +98,7 @@ public List getBlacklist() { } public String getPrerenderServiceUrl() { - final String prerenderServiceUrl = filterConfig.getInitParameter("prerenderServiceUrl"); + final String prerenderServiceUrl = config.get("prerenderServiceUrl"); return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : "http://service.prerender.io/"; } } diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java new file mode 100644 index 0000000..79f6a61 --- /dev/null +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -0,0 +1,339 @@ +package com.github.greengerong; + +import com.google.common.base.Predicate; +import org.apache.commons.lang.StringUtils; +import org.apache.http.*; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIUtils; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicHeader; +import org.apache.http.message.HeaderGroup; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.Closeable; +import java.io.IOException; +import java.io.PrintWriter; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Enumeration; +import java.util.List; +import java.util.regex.Pattern; + +import static com.google.common.collect.FluentIterable.from; + +class PrerenderSeoService { + + public static final int HTTP_OK = 200; + private CloseableHttpClient httpClient; + private PrerenderConfig prerenderConfig; + private PreRenderEventHandler preRenderEventHandler; + + /** + * These are the "hop-by-hop" headers that should not be copied. + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html + * I use an HttpClient HeaderGroup class instead of Set because this + * approach does case insensitive lookup faster. + */ + protected static final HeaderGroup hopByHopHeaders; + + public PrerenderSeoService(PrerenderConfig prerenderConfig) { + this.prerenderConfig = prerenderConfig; + this.httpClient = getHttpClient(); + } + + static { + hopByHopHeaders = new HeaderGroup(); + String[] headers = new String[]{ + "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", + "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; + for (String header : headers) { + hopByHopHeaders.addHeader(new BasicHeader(header, null)); + } + } + + public void destroy() { + if (preRenderEventHandler != null) { + preRenderEventHandler.destroy(); + } + closeQuietly(httpClient); + } + + public boolean tryPrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) { + try { + if (handlePrerender(servletRequest, servletResponse)) + return true; + } catch (Exception e) { + log.error("Prerender service error", e); + } + return false; + } + + private boolean handlePrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) + throws URISyntaxException, IOException { + if (shouldShowPrerenderedPage(servletRequest)) { + this.preRenderEventHandler = prerenderConfig.getEventHandler(); + if (beforeRender(servletRequest, servletResponse) || proxyPrerenderedPageResponse(servletRequest, servletResponse)) { + return true; + } + } + return false; + } + + private boolean hasEscapedFragment(HttpServletRequest request) { + return request.getParameterMap().containsKey("_escaped_fragment_"); + } + + public String getApiUrl(String url) { + String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); + if (!prerenderServiceUrl.endsWith("/")) { + prerenderServiceUrl += "/"; + } + return prerenderServiceUrl + url; + } + + private boolean isInBlackList(final String url, final String referer, List blacklist) { + return from(blacklist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + final Pattern pattern = Pattern.compile(regex); + return pattern.matcher(url).matches() || + (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); + } + }); + } + + private boolean isInSearchUserAgent(final String userAgent) { + return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return userAgent.toLowerCase().contains(item.toLowerCase()); + } + }); + } + + private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); + + public boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { + final String userAgent = request.getHeader("User-Agent"); + final String url = getRequestURL(request); + final String referer = request.getHeader("Referer"); + + log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); + + if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { + log.trace("Request is not HTTP GET; intercept: no"); + return false; + } + + if (isInResources(url)) { + log.trace("request is for a (static) resource; intercept: no"); + return false; + } + + final List whiteList = prerenderConfig.getWhitelist(); + if (whiteList != null && !isInWhiteList(url, whiteList)) { + log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); + return false; + } + + final List blacklist = prerenderConfig.getBlacklist(); + if (blacklist != null && isInBlackList(url, referer, blacklist)) { + log.trace("Blacklist is enabled, and this request is listed; intercept: no"); + return false; + } + + if (hasEscapedFragment(request)) { + log.trace("Request Has _escaped_fragment_; intercept: yes"); + return true; + } + + if (StringUtils.isBlank(userAgent)) { + log.trace("Request has blank userAgent; intercept: no"); + return false; + } + + if (!isInSearchUserAgent(userAgent)) { + log.trace("Request User-Agent is not a search bot; intercept: no"); + return false; + } + + log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); + return true; + } + + + public String getRequestURL(HttpServletRequest request) { + if (prerenderConfig.getForwardedURLHeader() != null) { + String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); + if (url != null) { + return url; + } + } + return request.getRequestURL().toString(); + } + + private boolean isInResources(final String url) { + return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return url.contains(item.toLowerCase()); + } + }); + } + + private boolean isInWhiteList(final String url, List whitelist) { + return from(whitelist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + return Pattern.compile(regex).matcher(url).matches(); + } + }); + } + + private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) throws IOException { + if (preRenderEventHandler != null) { + final String html = preRenderEventHandler.beforeRender(request); + if (StringUtils.isNotBlank(html)) { + final PrintWriter writer = response.getWriter(); + writer.write(html); + writer.flush(); + closeQuietly(writer); + return true; + } + } + return false; + } + + private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) + throws IOException, URISyntaxException { + final String apiUrl = getApiUrl(getFullUrl(request)); + log.trace(String.format("Prerender proxy will send request to:%s", apiUrl)); + final HttpGet getMethod = getHttpGet(apiUrl); + copyRequestHeaders(request, getMethod); + withPrerenderToken(getMethod); + CloseableHttpResponse proxyResponse = null; + + try { + proxyResponse = httpClient.execute(getMethod); + if (proxyResponse.getStatusLine().getStatusCode() == HTTP_OK) { + copyResponseHeaders(proxyResponse, response); + final String html = copyResponseEntity(proxyResponse, response); + afterRender(request, proxyResponse, html); + return true; + } + } finally { + closeQuietly(proxyResponse); + } + return false; + } + + private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse, String html) { + if (preRenderEventHandler != null) { + preRenderEventHandler.afterRender(request, proxyResponse, html); + } + } + + private void withPrerenderToken(HttpRequest proxyRequest) { + final String token = prerenderConfig.getPrerenderToken(); + //for new version prerender with token. + if (StringUtils.isNotBlank(token)) { + proxyRequest.addHeader("X-Prerender-Token", token); + } + } + + /** + * Copy proxied response headers back to the servlet client. + */ + protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { + for (Header header : proxyResponse.getAllHeaders()) { + if (!hopByHopHeaders.containsHeader(header.getName())) { + servletResponse.addHeader(header.getName(), header.getValue()); + } + } + } + + /** + * Copy response body data (the entity) from the proxy to the servlet client. + */ + protected String copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) + throws IOException { + HttpEntity entity = proxyResponse.getEntity(); + if (entity != null) { + PrintWriter printWriter = servletResponse.getWriter(); + try { + final String html = EntityUtils.toString(entity); + printWriter.write(html); + printWriter.flush(); + return html; + } finally { + closeQuietly(printWriter); + } + } + return ""; + } + + protected void closeQuietly(Closeable closeable) { + try { + if (closeable != null) { + closeable.close(); + } + } catch (IOException e) { + log.error("Close proxy error", e); + } + } + + private String getFullUrl(HttpServletRequest request) { + final String url = getRequestURL(request); + final String queryString = request.getQueryString(); + if (queryString != null) { + return url + '?' + queryString; + } + return url; + } + + /** + * Copy request headers from the servlet client to the proxy request. + * + * @throws java.net.URISyntaxException + */ + protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) + throws URISyntaxException { + // Get an Enumeration of all of the header names sent by the client + Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); + while (enumerationOfHeaderNames.hasMoreElements()) { + String headerName = (String) enumerationOfHeaderNames.nextElement(); + //Instead the content-length is effectively set via InputStreamEntity + if (!headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { + Enumeration headers = servletRequest.getHeaders(headerName); + while (headers.hasMoreElements()) {//sometimes more than one value + String headerValue = (String) headers.nextElement(); + // In case the proxy host is running multiple virtual servers, + // rewrite the Host header to ensure that we get content from + // the correct virtual server + if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { + HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); + headerValue = host.getHostName(); + if (host.getPort() != -1) { + headerValue += ":" + host.getPort(); + } + } + proxyRequest.addHeader(headerName, headerValue); + } + } + } + } + + + protected HttpGet getHttpGet(String apiUrl) { + return new HttpGet(apiUrl); + } + + protected CloseableHttpClient getHttpClient() { + return prerenderConfig.getHttpClient(); + } +} diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index d33edff..5fb1dfa 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -52,7 +52,7 @@ public void setUp() throws Exception { preRenderSEOFilter = new PreRenderSEOFilter() { @Override public void init(FilterConfig filterConfig) throws ServletException { - setPrerenderService(new PrerenderService(new PrerenderConfig(filterConfig)) { + setPrerenderSeoService(new PrerenderSeoService(new PrerenderConfig(toMap(filterConfig))) { @Override protected CloseableHttpClient getHttpClient() { return httpClient; @@ -97,7 +97,7 @@ public void should_handle_when_url_with_escaped_fragment_() throws Exception { final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -199,7 +199,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when @@ -262,7 +262,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when @@ -295,7 +295,7 @@ public void should_use_request_url_from_custom_header_if_available() throws Exce final HashMap map = Maps.newHashMap(); map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when From 9803d6264a62e5da6f41639389c535f0b904499a Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Wed, 28 May 2014 22:30:21 +0300 Subject: [PATCH 11/50] reorganizing and better naming --- .../greengerong/PreRenderSEOFilter.java | 2 +- .../greengerong/PrerenderSeoService.java | 242 +++++++++--------- 2 files changed, 122 insertions(+), 122 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 21963b1..0984b9e 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -24,7 +24,7 @@ public void init(FilterConfig filterConfig) throws ServletException { @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - boolean isPrerendered = prerenderSeoService.tryPrerender( + boolean isPrerendered = prerenderSeoService.prerenderIfEligible( (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); if (!isPrerendered) { filterChain.doFilter(servletRequest, servletResponse); diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 79f6a61..45c60e1 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -27,6 +27,7 @@ import static com.google.common.collect.FluentIterable.from; class PrerenderSeoService { + private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); public static final int HTTP_OK = 200; private CloseableHttpClient httpClient; @@ -39,7 +40,7 @@ class PrerenderSeoService { * I use an HttpClient HeaderGroup class instead of Set because this * approach does case insensitive lookup faster. */ - protected static final HeaderGroup hopByHopHeaders; + private static final HeaderGroup hopByHopHeaders; public PrerenderSeoService(PrerenderConfig prerenderConfig) { this.prerenderConfig = prerenderConfig; @@ -63,10 +64,11 @@ public void destroy() { closeQuietly(httpClient); } - public boolean tryPrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) { + public boolean prerenderIfEligible(HttpServletRequest servletRequest, HttpServletResponse servletResponse) { try { - if (handlePrerender(servletRequest, servletResponse)) + if (handlePrerender(servletRequest, servletResponse)) { return true; + } } catch (Exception e) { log.error("Prerender service error", e); } @@ -84,41 +86,7 @@ private boolean handlePrerender(HttpServletRequest servletRequest, HttpServletRe return false; } - private boolean hasEscapedFragment(HttpServletRequest request) { - return request.getParameterMap().containsKey("_escaped_fragment_"); - } - - public String getApiUrl(String url) { - String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); - if (!prerenderServiceUrl.endsWith("/")) { - prerenderServiceUrl += "/"; - } - return prerenderServiceUrl + url; - } - - private boolean isInBlackList(final String url, final String referer, List blacklist) { - return from(blacklist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - final Pattern pattern = Pattern.compile(regex); - return pattern.matcher(url).matches() || - (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); - } - }); - } - - private boolean isInSearchUserAgent(final String userAgent) { - return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return userAgent.toLowerCase().contains(item.toLowerCase()); - } - }); - } - - private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); - - public boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { + private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { final String userAgent = request.getHeader("User-Agent"); final String url = getRequestURL(request); final String referer = request.getHeader("Referer"); @@ -166,8 +134,47 @@ public boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URIS return true; } + protected HttpGet getHttpGet(String apiUrl) { + return new HttpGet(apiUrl); + } + + protected CloseableHttpClient getHttpClient() { + return prerenderConfig.getHttpClient(); + } + + /** + * Copy request headers from the servlet client to the proxy request. + * + * @throws java.net.URISyntaxException + */ + private void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) + throws URISyntaxException { + // Get an Enumeration of all of the header names sent by the client + Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); + while (enumerationOfHeaderNames.hasMoreElements()) { + String headerName = (String) enumerationOfHeaderNames.nextElement(); + //Instead the content-length is effectively set via InputStreamEntity + if (!headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { + Enumeration headers = servletRequest.getHeaders(headerName); + while (headers.hasMoreElements()) {//sometimes more than one value + String headerValue = (String) headers.nextElement(); + // In case the proxy host is running multiple virtual servers, + // rewrite the Host header to ensure that we get content from + // the correct virtual server + if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { + HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); + headerValue = host.getHostName(); + if (host.getPort() != -1) { + headerValue += ":" + host.getPort(); + } + } + proxyRequest.addHeader(headerName, headerValue); + } + } + } + } - public String getRequestURL(HttpServletRequest request) { + private String getRequestURL(HttpServletRequest request) { if (prerenderConfig.getForwardedURLHeader() != null) { String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); if (url != null) { @@ -177,6 +184,81 @@ public String getRequestURL(HttpServletRequest request) { return request.getRequestURL().toString(); } + private String getApiUrl(String url) { + String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); + if (!prerenderServiceUrl.endsWith("/")) { + prerenderServiceUrl += "/"; + } + return prerenderServiceUrl + url; + } + + /** + * Copy proxied response headers back to the servlet client. + */ + private void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { + for (Header header : proxyResponse.getAllHeaders()) { + if (!hopByHopHeaders.containsHeader(header.getName())) { + servletResponse.addHeader(header.getName(), header.getValue()); + } + } + } + + /** + * Copy response body data (the entity) from the proxy to the servlet client. + */ + private String copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) + throws IOException { + HttpEntity entity = proxyResponse.getEntity(); + if (entity != null) { + PrintWriter printWriter = servletResponse.getWriter(); + try { + final String html = EntityUtils.toString(entity); + printWriter.write(html); + printWriter.flush(); + return html; + } finally { + closeQuietly(printWriter); + } + } + return ""; + } + + + protected void closeQuietly(Closeable closeable) { + try { + if (closeable != null) { + closeable.close(); + } + } catch (IOException e) { + log.error("Close proxy error", e); + } + } + + private boolean hasEscapedFragment(HttpServletRequest request) { + return request.getParameterMap().containsKey("_escaped_fragment_"); + } + + private boolean isInBlackList(final String url, final String referer, List blacklist) { + return from(blacklist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + final Pattern pattern = Pattern.compile(regex); + return pattern.matcher(url).matches() || + (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); + } + }); + } + + private boolean isInSearchUserAgent(final String userAgent) { + return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return userAgent.toLowerCase().contains(item.toLowerCase()); + } + }); + } + + private boolean isInResources(final String url) { return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { @Override @@ -246,47 +328,6 @@ private void withPrerenderToken(HttpRequest proxyRequest) { } } - /** - * Copy proxied response headers back to the servlet client. - */ - protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { - for (Header header : proxyResponse.getAllHeaders()) { - if (!hopByHopHeaders.containsHeader(header.getName())) { - servletResponse.addHeader(header.getName(), header.getValue()); - } - } - } - - /** - * Copy response body data (the entity) from the proxy to the servlet client. - */ - protected String copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) - throws IOException { - HttpEntity entity = proxyResponse.getEntity(); - if (entity != null) { - PrintWriter printWriter = servletResponse.getWriter(); - try { - final String html = EntityUtils.toString(entity); - printWriter.write(html); - printWriter.flush(); - return html; - } finally { - closeQuietly(printWriter); - } - } - return ""; - } - - protected void closeQuietly(Closeable closeable) { - try { - if (closeable != null) { - closeable.close(); - } - } catch (IOException e) { - log.error("Close proxy error", e); - } - } - private String getFullUrl(HttpServletRequest request) { final String url = getRequestURL(request); final String queryString = request.getQueryString(); @@ -295,45 +336,4 @@ private String getFullUrl(HttpServletRequest request) { } return url; } - - /** - * Copy request headers from the servlet client to the proxy request. - * - * @throws java.net.URISyntaxException - */ - protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) - throws URISyntaxException { - // Get an Enumeration of all of the header names sent by the client - Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); - while (enumerationOfHeaderNames.hasMoreElements()) { - String headerName = (String) enumerationOfHeaderNames.nextElement(); - //Instead the content-length is effectively set via InputStreamEntity - if (!headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { - Enumeration headers = servletRequest.getHeaders(headerName); - while (headers.hasMoreElements()) {//sometimes more than one value - String headerValue = (String) headers.nextElement(); - // In case the proxy host is running multiple virtual servers, - // rewrite the Host header to ensure that we get content from - // the correct virtual server - if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { - HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); - headerValue = host.getHostName(); - if (host.getPort() != -1) { - headerValue += ":" + host.getPort(); - } - } - proxyRequest.addHeader(headerName, headerValue); - } - } - } - } - - - protected HttpGet getHttpGet(String apiUrl) { - return new HttpGet(apiUrl); - } - - protected CloseableHttpClient getHttpClient() { - return prerenderConfig.getHttpClient(); - } } From a2ed50bbcf2f0946e9fdfef5665edbaee73cc2b4 Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Wed, 28 May 2014 22:34:56 +0300 Subject: [PATCH 12/50] rearrange class entries --- .../com/github/greengerong/PrerenderSeoService.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 45c60e1..141691c 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -27,13 +27,8 @@ import static com.google.common.collect.FluentIterable.from; class PrerenderSeoService { - private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); - public static final int HTTP_OK = 200; - private CloseableHttpClient httpClient; - private PrerenderConfig prerenderConfig; - private PreRenderEventHandler preRenderEventHandler; - + private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); /** * These are the "hop-by-hop" headers that should not be copied. * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html @@ -41,6 +36,9 @@ class PrerenderSeoService { * approach does case insensitive lookup faster. */ private static final HeaderGroup hopByHopHeaders; + private CloseableHttpClient httpClient; + private PrerenderConfig prerenderConfig; + private PreRenderEventHandler preRenderEventHandler; public PrerenderSeoService(PrerenderConfig prerenderConfig) { this.prerenderConfig = prerenderConfig; From b1563745eaca8d36ce9c44ee794e8c5aa4e82bbe Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Wed, 28 May 2014 22:48:50 +0300 Subject: [PATCH 13/50] simplified PrerenderSeoService construction --- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 2 +- .../java/com/github/greengerong/PrerenderSeoService.java | 5 +++-- .../java/com/github/greengerong/PreRenderSEOFilterTest.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 0984b9e..de582a1 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -18,7 +18,7 @@ public class PreRenderSEOFilter implements Filter { @Override public void init(FilterConfig filterConfig) throws ServletException { - this.prerenderSeoService = new PrerenderSeoService(new PrerenderConfig(toMap(filterConfig))); + this.prerenderSeoService = new PrerenderSeoService(toMap(filterConfig)); } @Override diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 141691c..7face17 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -22,6 +22,7 @@ import java.net.URISyntaxException; import java.util.Enumeration; import java.util.List; +import java.util.Map; import java.util.regex.Pattern; import static com.google.common.collect.FluentIterable.from; @@ -40,8 +41,8 @@ class PrerenderSeoService { private PrerenderConfig prerenderConfig; private PreRenderEventHandler preRenderEventHandler; - public PrerenderSeoService(PrerenderConfig prerenderConfig) { - this.prerenderConfig = prerenderConfig; + public PrerenderSeoService(Map config) { + this.prerenderConfig = new PrerenderConfig(config); this.httpClient = getHttpClient(); } diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 5fb1dfa..2945053 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -52,7 +52,7 @@ public void setUp() throws Exception { preRenderSEOFilter = new PreRenderSEOFilter() { @Override public void init(FilterConfig filterConfig) throws ServletException { - setPrerenderSeoService(new PrerenderSeoService(new PrerenderConfig(toMap(filterConfig))) { + setPrerenderSeoService(new PrerenderSeoService(toMap(filterConfig)) { @Override protected CloseableHttpClient getHttpClient() { return httpClient; From c798d1f5f143798d1dcf199bc68008917e789f7d Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Wed, 28 May 2014 23:54:52 +0300 Subject: [PATCH 14/50] add missing public keyword --- src/main/java/com/github/greengerong/PrerenderSeoService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 7face17..2a34683 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -27,7 +27,7 @@ import static com.google.common.collect.FluentIterable.from; -class PrerenderSeoService { +public class PrerenderSeoService { public static final int HTTP_OK = 200; private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); /** From c698f44a1996c356ea85900748d00398d94acdbf Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Tue, 17 Jun 2014 15:16:40 +0300 Subject: [PATCH 15/50] add configurable socket timeout to http client --- .../github/greengerong/PrerenderConfig.java | 23 +++++++++++++-- .../greengerong/PrerenderConfigTest.java | 28 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/github/greengerong/PrerenderConfigTest.java diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 3407848..fc566f5 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -4,6 +4,7 @@ import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; +import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; @@ -38,15 +39,33 @@ public PreRenderEventHandler getEventHandler() { public CloseableHttpClient getHttpClient() { HttpClientBuilder builder = HttpClients.custom(); + builder = configureProxy(builder); + builder = configureTimeout(builder); + builder = builder.setConnectionManager(new PoolingHttpClientConnectionManager()); + return builder.build(); + } + + private HttpClientBuilder configureProxy(HttpClientBuilder builder) { final String proxy = config.get("proxy"); if (StringUtils.isNotBlank(proxy)) { final int proxyPort = Integer.parseInt(config.get("proxyPort")); DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); builder = builder.setRoutePlanner(routePlanner); } + return builder; + } + + private HttpClientBuilder configureTimeout(HttpClientBuilder builder) { + final String socketTimeout = getSocketTimeout(); + if (socketTimeout != null) { + RequestConfig config = RequestConfig.custom().setSocketTimeout(Integer.parseInt(socketTimeout)).build(); + builder = builder.setDefaultRequestConfig(config); + } + return builder; + } - builder = builder.setConnectionManager(new PoolingHttpClientConnectionManager()); - return builder.build(); + public String getSocketTimeout() { + return config.get("socketTimeout"); } public String getPrerenderToken() { diff --git a/src/test/java/com/github/greengerong/PrerenderConfigTest.java b/src/test/java/com/github/greengerong/PrerenderConfigTest.java new file mode 100644 index 0000000..3371603 --- /dev/null +++ b/src/test/java/com/github/greengerong/PrerenderConfigTest.java @@ -0,0 +1,28 @@ +package com.github.greengerong; + +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class PrerenderConfigTest { + @Test(expected = NumberFormatException.class) + public void should_throw_exception_if_invalid_timeout_value_specified() throws Exception { + //given + Map configuration = new HashMap(); + configuration.put("socketTimeout", "not_an_int"); + PrerenderConfig config = new PrerenderConfig(configuration); + //when + config.getHttpClient(); + } + + @Test + public void should_pass_if_correct_timeout_value_specified() throws Exception { + //given + Map configuration = new HashMap(); + configuration.put("socketTimeout", "1000"); + PrerenderConfig config = new PrerenderConfig(configuration); + //when + config.getHttpClient(); + } +} From b6d954097427181294d51c353bf64d0206b4b2a7 Mon Sep 17 00:00:00 2001 From: greengerong Date: Tue, 17 Jun 2014 23:17:18 +0800 Subject: [PATCH 16/50] [(Fix) #11]: add support for prerender-status-code --- pom.xml | 5 +++-- prerender-java.iml | 1 - .../com/github/greengerong/PrerenderSeoService.java | 12 +++++------- .../github/greengerong/PreRenderSEOFilterTest.java | 6 ++++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pom.xml b/pom.xml index 380a7a8..842c22c 100644 --- a/pom.xml +++ b/pom.xml @@ -16,6 +16,7 @@ http://www.apache.org/licenses/LICENSE-2.0.txt repo + scm:git:git@github.com:greengerong/prerender-java.git @@ -91,12 +92,12 @@ sonatype-nexus-snapshots Sonatype Nexus Snapshots - http://oss.sonatype.org/content/repositories/snapshots + https://oss.sonatype.org/content/repositories/snapshots sonatype-nexus-staging Nexus Release Repository - http://oss.sonatype.org/service/local/staging/deploy/maven2/ + https://oss.sonatype.org/service/local/staging/deploy/maven2/ diff --git a/prerender-java.iml b/prerender-java.iml index 968be0a..cbd3c69 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -4,7 +4,6 @@ - diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 2a34683..54fbe39 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -301,16 +301,14 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer try { proxyResponse = httpClient.execute(getMethod); - if (proxyResponse.getStatusLine().getStatusCode() == HTTP_OK) { - copyResponseHeaders(proxyResponse, response); - final String html = copyResponseEntity(proxyResponse, response); - afterRender(request, proxyResponse, html); - return true; - } + response.setStatus(proxyResponse.getStatusLine().getStatusCode()); + copyResponseHeaders(proxyResponse, response); + final String html = copyResponseEntity(proxyResponse, response); + afterRender(request, proxyResponse, html); + return true; } finally { closeQuietly(proxyResponse); } - return false; } private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse, String html) { diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 2945053..6ae1935 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -211,7 +211,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and } @Test - public void should_not_handle_when_every_thing_is_ok_but_prerender_server_response_is_not_200() throws Exception { + public void should_handle_when_every_thing_is_ok_but_prerender_server_response_is_not_200() throws Exception { //given when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); preRenderSEOFilter.init(filterConfig); @@ -230,13 +230,15 @@ public void should_not_handle_when_every_thing_is_ok_but_prerender_server_respon map.put("_escaped_fragment_", ""); when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(NOT_FOUND); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); //then verify(httpClient).execute(httpGet); - verify(filterChain).doFilter(servletRequest, servletResponse); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + verify(servletResponse).setStatus(NOT_FOUND); } From c8342fba58ed66d0f973d6becc7b62d83d4090de Mon Sep 17 00:00:00 2001 From: green Date: Tue, 17 Jun 2014 23:32:19 +0800 Subject: [PATCH 17/50] update version to 1.6.3-SNAPSHOT --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6c67fc5..19a1e53 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.6.2 + 1.6.3-SNAPSHOT ## How it works From 0b58e99c47ceb47edcc8733100eab59969adf548 Mon Sep 17 00:00:00 2001 From: "MD. Mohiuddin ahmed" Date: Sun, 13 Jul 2014 23:01:14 +0600 Subject: [PATCH 18/50] Added .xml in exclution list. Prerender renders xmls so Google SiteMap Test Fails because it gets www.testsite.com/sitemap.xml as an html and returns "Your Sitemap appears to be an HTML page. Please use a supported sitemap format instead". --- src/main/java/com/github/greengerong/PrerenderConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index fc566f5..16eeb6f 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -88,7 +88,7 @@ public List getCrawlerUserAgents() { } public List getExtensionsToIgnore() { - List extensionsToIgnore = Lists.newArrayList(".js", ".css", ".less", ".png", ".jpg", ".jpeg", + List extensionsToIgnore = Lists.newArrayList(".xml",".js", ".css", ".less", ".png", ".jpg", ".jpeg", ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent"); From 1507e84e27badb53cc75161f16051ebd655214c6 Mon Sep 17 00:00:00 2001 From: Richard Midwinter Date: Thu, 7 Aug 2014 21:20:17 +0100 Subject: [PATCH 19/50] Improved file extension checking. Updated logic to look for file extensions at the end of the URL, or just before the first question mark if one is present. This should prevent matching on subdomains or URL parameter values. --- src/main/java/com/github/greengerong/PrerenderSeoService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 54fbe39..e2e2d08 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -262,7 +262,8 @@ private boolean isInResources(final String url) { return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { @Override public boolean apply(String item) { - return url.contains(item.toLowerCase()); + return (url.indexOf('?') >= 0 ? url.substring(0, url.indexOf('?')) : url) + .toLowerCase().endsWith(item); } }); } From 8dd2cff8ea56f1a1765f8ba2afcea56db0260b1e Mon Sep 17 00:00:00 2001 From: greengerong Date: Wed, 13 Aug 2014 21:40:06 +0800 Subject: [PATCH 20/50] [maven-release-plugin] prepare release prerender-java-1.6.3 --- pom.xml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 842c22c..6abb782 100644 --- a/pom.xml +++ b/pom.xml @@ -1,10 +1,9 @@ - + 4.0.0 com.github.greengerong prerender-java - 1.6.3-SNAPSHOT + 1.6.3 jar prerender-java @@ -28,7 +27,7 @@ oss-parent 7 - + UTF-8 4.11 From 285d10f2a9c510dfe01623db672efaafd6488ad6 Mon Sep 17 00:00:00 2001 From: greengerong Date: Wed, 13 Aug 2014 21:40:19 +0800 Subject: [PATCH 21/50] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6abb782..112aab4 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.3 + 1.6.4-SNAPSHOT jar prerender-java From f3db14bf9e9f676a1e0f886a00a219a3744cdba2 Mon Sep 17 00:00:00 2001 From: greengerong Date: Sat, 16 Aug 2014 08:05:57 +0800 Subject: [PATCH 22/50] [(imporve) #21]: Getting jsessionids with urls. (Add response to event handler before reponse prerender html) --- .../greengerong/PreRenderEventHandler.java | 3 +- .../greengerong/PrerenderSeoService.java | 49 ++++++++++--------- .../greengerong/PreRenderSEOFilterTest.java | 12 ++++- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderEventHandler.java b/src/main/java/com/github/greengerong/PreRenderEventHandler.java index 9669afb..f1f6022 100644 --- a/src/main/java/com/github/greengerong/PreRenderEventHandler.java +++ b/src/main/java/com/github/greengerong/PreRenderEventHandler.java @@ -4,12 +4,13 @@ import org.apache.http.HttpResponse; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; public interface PreRenderEventHandler { String beforeRender(HttpServletRequest clientRequest); - void afterRender(HttpServletRequest clientRequest, HttpResponse prerenderResponse, String html); + void afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, HttpResponse prerenderResponse, String responseHtml); void destroy(); } diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index e2e2d08..176d128 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -202,24 +202,24 @@ private void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse } } + private String getResponseHtml(HttpResponse proxyResponse) + throws IOException { + HttpEntity entity = proxyResponse.getEntity(); + return entity != null ? EntityUtils.toString(entity) : ""; + } + /** * Copy response body data (the entity) from the proxy to the servlet client. */ - private String copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) + private void responseEntity(String html, HttpServletResponse servletResponse) throws IOException { - HttpEntity entity = proxyResponse.getEntity(); - if (entity != null) { - PrintWriter printWriter = servletResponse.getWriter(); - try { - final String html = EntityUtils.toString(entity); - printWriter.write(html); - printWriter.flush(); - return html; - } finally { - closeQuietly(printWriter); - } + PrintWriter printWriter = servletResponse.getWriter(); + try { + printWriter.write(html); + printWriter.flush(); + } finally { + closeQuietly(printWriter); } - return ""; } @@ -262,8 +262,8 @@ private boolean isInResources(final String url) { return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { @Override public boolean apply(String item) { - return (url.indexOf('?') >= 0 ? url.substring(0, url.indexOf('?')) : url) - .toLowerCase().endsWith(item); + return (url.indexOf('?') >= 0 ? url.substring(0, url.indexOf('?')) : url) + .toLowerCase().endsWith(item); } }); } @@ -298,23 +298,24 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer final HttpGet getMethod = getHttpGet(apiUrl); copyRequestHeaders(request, getMethod); withPrerenderToken(getMethod); - CloseableHttpResponse proxyResponse = null; + CloseableHttpResponse prerenderServerResponse = null; try { - proxyResponse = httpClient.execute(getMethod); - response.setStatus(proxyResponse.getStatusLine().getStatusCode()); - copyResponseHeaders(proxyResponse, response); - final String html = copyResponseEntity(proxyResponse, response); - afterRender(request, proxyResponse, html); + prerenderServerResponse = httpClient.execute(getMethod); + response.setStatus(prerenderServerResponse.getStatusLine().getStatusCode()); + copyResponseHeaders(prerenderServerResponse, response); + final String html = getResponseHtml(prerenderServerResponse); + afterRender(request, response, prerenderServerResponse, html); + responseEntity(html, response); return true; } finally { - closeQuietly(proxyResponse); + closeQuietly(prerenderServerResponse); } } - private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse, String html) { + private void afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, CloseableHttpResponse prerenderServerResponse, String responseHtml) { if (preRenderEventHandler != null) { - preRenderEventHandler.afterRender(request, proxyResponse, html); + preRenderEventHandler.afterRender(clientRequest, clientResponse, prerenderServerResponse, responseHtml); } } diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 6ae1935..e4c935d 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -18,6 +18,8 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.Enumeration; import java.util.HashMap; @@ -46,6 +48,8 @@ public class PreRenderSEOFilterTest { @Mock private HttpGet httpGet; + @Mock + private PrintWriter printWriter; @Before public void setUp() throws Exception { @@ -99,6 +103,8 @@ public void should_handle_when_url_with_escaped_fragment_() throws Exception { when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -201,7 +207,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); - + when(servletResponse.getWriter()).thenReturn(printWriter); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -231,6 +237,8 @@ public void should_handle_when_every_thing_is_ok_but_prerender_server_response_i when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(NOT_FOUND); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -266,6 +274,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -299,6 +308,7 @@ public void should_use_request_url_from_custom_header_if_available() throws Exce when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); From 3ce13c6f5eb840423088f6e9267fba7b4a5f94a2 Mon Sep 17 00:00:00 2001 From: greengerong Date: Sat, 16 Aug 2014 08:19:31 +0800 Subject: [PATCH 23/50] update version in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 19a1e53..7f71558 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.6.3-SNAPSHOT + 1.6.4-SNAPSHOT ## How it works From a27db6ba0c92a1c3e018f0e52f5c05de18993fc8 Mon Sep 17 00:00:00 2001 From: greengerong Date: Tue, 26 Aug 2014 22:56:51 +0800 Subject: [PATCH 24/50] [tech] response html can be changed by event handler after render --- .../com/github/greengerong/PreRenderEventHandler.java | 2 +- .../java/com/github/greengerong/PrerenderSeoService.java | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderEventHandler.java b/src/main/java/com/github/greengerong/PreRenderEventHandler.java index f1f6022..bd71634 100644 --- a/src/main/java/com/github/greengerong/PreRenderEventHandler.java +++ b/src/main/java/com/github/greengerong/PreRenderEventHandler.java @@ -10,7 +10,7 @@ public interface PreRenderEventHandler { String beforeRender(HttpServletRequest clientRequest); - void afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, HttpResponse prerenderResponse, String responseHtml); + String afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, HttpResponse prerenderResponse, String responseHtml); void destroy(); } diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 176d128..22445b9 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -304,8 +304,8 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer prerenderServerResponse = httpClient.execute(getMethod); response.setStatus(prerenderServerResponse.getStatusLine().getStatusCode()); copyResponseHeaders(prerenderServerResponse, response); - final String html = getResponseHtml(prerenderServerResponse); - afterRender(request, response, prerenderServerResponse, html); + String html = getResponseHtml(prerenderServerResponse); + html = afterRender(request, response, prerenderServerResponse, html); responseEntity(html, response); return true; } finally { @@ -313,10 +313,11 @@ private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpSer } } - private void afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, CloseableHttpResponse prerenderServerResponse, String responseHtml) { + private String afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, CloseableHttpResponse prerenderServerResponse, String responseHtml) { if (preRenderEventHandler != null) { - preRenderEventHandler.afterRender(clientRequest, clientResponse, prerenderServerResponse, responseHtml); + return preRenderEventHandler.afterRender(clientRequest, clientResponse, prerenderServerResponse, responseHtml); } + return responseHtml; } private void withPrerenderToken(HttpRequest proxyRequest) { From 7cd89b8cd1f2cb81a7e460815e749ee1549b5842 Mon Sep 17 00:00:00 2001 From: green Date: Mon, 29 Sep 2014 07:45:22 +0800 Subject: [PATCH 25/50] Update README.md --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7f71558..50d9bb9 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,8 @@ Use this java filter that prerenders a javascript-rendered page using an externa prerender com.github.greengerong.PreRenderSEOFilter - prerenderServiceUrl - http://localhost:3000 - - - crawlerUserAgents - me + prerenderToken + [get from prerender: https://prerender.io/] From 4febb8e481240f88aefb966bee79526ac8f5fba0 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 13 Nov 2014 14:28:42 +0800 Subject: [PATCH 26/50] [Tech] fix test, make the test to clean --- .../com/github/greengerong/PrerenderConfigTest.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/github/greengerong/PrerenderConfigTest.java b/src/test/java/com/github/greengerong/PrerenderConfigTest.java index 3371603..d144e85 100644 --- a/src/test/java/com/github/greengerong/PrerenderConfigTest.java +++ b/src/test/java/com/github/greengerong/PrerenderConfigTest.java @@ -1,12 +1,17 @@ package com.github.greengerong; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Test; import java.util.HashMap; import java.util.Map; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsNull.notNullValue; +import static org.junit.Assert.assertThat; + public class PrerenderConfigTest { - @Test(expected = NumberFormatException.class) + @Test(expected = Exception.class) public void should_throw_exception_if_invalid_timeout_value_specified() throws Exception { //given Map configuration = new HashMap(); @@ -23,6 +28,8 @@ public void should_pass_if_correct_timeout_value_specified() throws Exception { configuration.put("socketTimeout", "1000"); PrerenderConfig config = new PrerenderConfig(configuration); //when - config.getHttpClient(); + final CloseableHttpClient httpClient = config.getHttpClient(); + + assertThat(httpClient, is(notNullValue())); } } From d401f984252e81865a525cf35ca35457b3ba6914 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 13 Nov 2014 17:02:41 +0800 Subject: [PATCH 27/50] [Tech] clean code. --- .../greengerong/PreRenderSEOFilter.java | 3 +- .../github/greengerong/PrerenderConfig.java | 32 ++++++++------- .../greengerong/PrerenderSeoService.java | 39 ++++++++++++------- .../greengerong/PreRenderSEOFilterTest.java | 21 +++++----- 4 files changed, 55 insertions(+), 40 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index de582a1..6eb721f 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,6 +1,7 @@ package com.github.greengerong; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import javax.servlet.*; import javax.servlet.http.HttpServletRequest; @@ -41,7 +42,7 @@ protected void setPrerenderSeoService(PrerenderSeoService prerenderSeoService) { } protected Map toMap(FilterConfig filterConfig) { - HashMap config = new HashMap(); + Map config = Maps.newHashMap(); for (String parameterName : PARAMETER_NAMES) { config.put(parameterName, filterConfig.getInitParameter(parameterName)); } diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 16eeb6f..1219486 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -17,8 +17,11 @@ import java.util.List; import java.util.Map; +import static org.apache.commons.lang.StringUtils.isNotBlank; + public class PrerenderConfig { private final static Logger log = LoggerFactory.getLogger(PrerenderConfig.class); + public static final String PRERENDER_IO_SERVICE_URL = "http://service.prerender.io/"; private Map config; public PrerenderConfig(Map config) { @@ -27,7 +30,7 @@ public PrerenderConfig(Map config) { public PreRenderEventHandler getEventHandler() { final String preRenderEventHandler = config.get("preRenderEventHandler"); - if (StringUtils.isNotBlank(preRenderEventHandler)) { + if (isNotBlank(preRenderEventHandler)) { try { return (PreRenderEventHandler) Class.forName(preRenderEventHandler).newInstance(); } catch (Exception e) { @@ -38,19 +41,20 @@ public PreRenderEventHandler getEventHandler() { } public CloseableHttpClient getHttpClient() { - HttpClientBuilder builder = HttpClients.custom(); - builder = configureProxy(builder); - builder = configureTimeout(builder); - builder = builder.setConnectionManager(new PoolingHttpClientConnectionManager()); + HttpClientBuilder builder = HttpClients.custom() + .setConnectionManager(new PoolingHttpClientConnectionManager()); + + configureProxy(builder); + configureTimeout(builder); return builder.build(); } private HttpClientBuilder configureProxy(HttpClientBuilder builder) { final String proxy = config.get("proxy"); - if (StringUtils.isNotBlank(proxy)) { + if (isNotBlank(proxy)) { final int proxyPort = Integer.parseInt(config.get("proxyPort")); DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); - builder = builder.setRoutePlanner(routePlanner); + builder.setRoutePlanner(routePlanner); } return builder; } @@ -59,7 +63,7 @@ private HttpClientBuilder configureTimeout(HttpClientBuilder builder) { final String socketTimeout = getSocketTimeout(); if (socketTimeout != null) { RequestConfig config = RequestConfig.custom().setSocketTimeout(Integer.parseInt(socketTimeout)).build(); - builder = builder.setDefaultRequestConfig(config); + builder.setDefaultRequestConfig(config); } return builder; } @@ -80,7 +84,7 @@ public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly"); final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); - if (StringUtils.isNotBlank(crawlerUserAgentsFromConfig)) { + if (isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); } @@ -88,12 +92,12 @@ public List getCrawlerUserAgents() { } public List getExtensionsToIgnore() { - List extensionsToIgnore = Lists.newArrayList(".xml",".js", ".css", ".less", ".png", ".jpg", ".jpeg", + List extensionsToIgnore = Lists.newArrayList(".xml", ".js", ".css", ".less", ".png", ".jpg", ".jpeg", ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent"); final String extensionsToIgnoreFromConfig = config.get("extensionsToIgnore"); - if (StringUtils.isNotBlank(extensionsToIgnoreFromConfig)) { + if (isNotBlank(extensionsToIgnoreFromConfig)) { extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); } @@ -102,7 +106,7 @@ public List getExtensionsToIgnore() { public List getWhitelist() { final String whitelist = config.get("whitelist"); - if (StringUtils.isNotBlank(whitelist)) { + if (isNotBlank(whitelist)) { return Arrays.asList(whitelist.trim().split(",")); } return null; @@ -110,7 +114,7 @@ public List getWhitelist() { public List getBlacklist() { final String blacklist = config.get("blacklist"); - if (StringUtils.isNotBlank(blacklist)) { + if (isNotBlank(blacklist)) { return Arrays.asList(blacklist.trim().split(",")); } return null; @@ -118,6 +122,6 @@ public List getBlacklist() { public String getPrerenderServiceUrl() { final String prerenderServiceUrl = config.get("prerenderServiceUrl"); - return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : "http://service.prerender.io/"; + return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : PRERENDER_IO_SERVICE_URL; } } diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 22445b9..a59b697 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -1,5 +1,6 @@ package com.github.greengerong; +import com.google.common.base.Function; import com.google.common.base.Predicate; import org.apache.commons.lang.StringUtils; import org.apache.http.*; @@ -20,15 +21,18 @@ import java.io.PrintWriter; import java.net.URI; import java.net.URISyntaxException; +import java.util.Arrays; import java.util.Enumeration; import java.util.List; import java.util.Map; import java.util.regex.Pattern; import static com.google.common.collect.FluentIterable.from; +import static org.apache.commons.lang.StringUtils.isNotBlank; +import static org.apache.http.HttpHeaders.CONTENT_LENGTH; +import static org.apache.http.HttpHeaders.HOST; public class PrerenderSeoService { - public static final int HTTP_OK = 200; private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); /** * These are the "hop-by-hop" headers that should not be copied. @@ -37,6 +41,7 @@ public class PrerenderSeoService { * approach does case insensitive lookup faster. */ private static final HeaderGroup hopByHopHeaders; + public static final String ESCAPED_FRAGMENT_KEY = "_escaped_fragment_"; private CloseableHttpClient httpClient; private PrerenderConfig prerenderConfig; private PreRenderEventHandler preRenderEventHandler; @@ -90,7 +95,7 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI final String url = getRequestURL(request); final String referer = request.getHeader("Referer"); - log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); + log.trace(String.format("checking request for %s from User-Agent %s and referer %s", url, userAgent, referer)); if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { log.trace("Request is not HTTP GET; intercept: no"); @@ -153,14 +158,14 @@ private void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest p while (enumerationOfHeaderNames.hasMoreElements()) { String headerName = (String) enumerationOfHeaderNames.nextElement(); //Instead the content-length is effectively set via InputStreamEntity - if (!headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { + if (!headerName.equalsIgnoreCase(CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { Enumeration headers = servletRequest.getHeaders(headerName); while (headers.hasMoreElements()) {//sometimes more than one value String headerValue = (String) headers.nextElement(); // In case the proxy host is running multiple virtual servers, // rewrite the Host header to ensure that we get content from // the correct virtual server - if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { + if (headerName.equalsIgnoreCase(HOST)) { HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); headerValue = host.getHostName(); if (host.getPort() != -1) { @@ -194,12 +199,19 @@ private String getApiUrl(String url) { /** * Copy proxied response headers back to the servlet client. */ - private void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { - for (Header header : proxyResponse.getAllHeaders()) { - if (!hopByHopHeaders.containsHeader(header.getName())) { + private void copyResponseHeaders(HttpResponse proxyResponse, final HttpServletResponse servletResponse) { + from(Arrays.asList(proxyResponse.getAllHeaders())).filter(new Predicate
() { + @Override + public boolean apply(Header header) { + return !hopByHopHeaders.containsHeader(header.getName()); + } + }).transform(new Function() { + @Override + public Boolean apply(Header header) { servletResponse.addHeader(header.getName(), header.getValue()); + return true; } - } + }).toList(); } private String getResponseHtml(HttpResponse proxyResponse) @@ -234,7 +246,7 @@ protected void closeQuietly(Closeable closeable) { } private boolean hasEscapedFragment(HttpServletRequest request) { - return request.getParameterMap().containsKey("_escaped_fragment_"); + return request.getParameterMap().containsKey(ESCAPED_FRAGMENT_KEY); } private boolean isInBlackList(final String url, final String referer, List blacklist) { @@ -280,7 +292,7 @@ public boolean apply(String regex) { private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) throws IOException { if (preRenderEventHandler != null) { final String html = preRenderEventHandler.beforeRender(request); - if (StringUtils.isNotBlank(html)) { + if (isNotBlank(html)) { final PrintWriter writer = response.getWriter(); writer.write(html); writer.flush(); @@ -323,7 +335,7 @@ private String afterRender(HttpServletRequest clientRequest, HttpServletResponse private void withPrerenderToken(HttpRequest proxyRequest) { final String token = prerenderConfig.getPrerenderToken(); //for new version prerender with token. - if (StringUtils.isNotBlank(token)) { + if (isNotBlank(token)) { proxyRequest.addHeader("X-Prerender-Token", token); } } @@ -331,9 +343,6 @@ private void withPrerenderToken(HttpRequest proxyRequest) { private String getFullUrl(HttpServletRequest request) { final String url = getRequestURL(request); final String queryString = request.getQueryString(); - if (queryString != null) { - return url + '?' + queryString; - } - return url; + return isNotBlank(queryString) ? String.format("%s?%s", url, queryString) : url; } } diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index e4c935d..49229c2 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -19,10 +19,11 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.PrintWriter; -import java.io.StringWriter; import java.util.Enumeration; import java.util.HashMap; +import static com.github.greengerong.PrerenderSeoService.ESCAPED_FRAGMENT_KEY; +import static org.apache.http.HttpStatus.SC_OK; import static org.mockito.Mockito.*; @RunWith(MockitoJUnitRunner.class) @@ -99,9 +100,9 @@ public void should_handle_when_url_with_escaped_fragment_() throws Exception { when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(SC_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); when(servletResponse.getWriter()).thenReturn(printWriter); @@ -203,9 +204,9 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(SC_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); when(servletResponse.getWriter()).thenReturn(printWriter); //when @@ -233,7 +234,7 @@ public void should_handle_when_every_thing_is_ok_but_prerender_server_response_i when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); when(statusLine.getStatusCode()).thenReturn(NOT_FOUND); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); @@ -270,9 +271,9 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(SC_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); when(servletResponse.getWriter()).thenReturn(printWriter); @@ -304,9 +305,9 @@ public void should_use_request_url_from_custom_header_if_available() throws Exce when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PrerenderSeoService.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(SC_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); when(servletResponse.getWriter()).thenReturn(printWriter); From f011cc926c5105d38f0988ba8797f863699f9d5e Mon Sep 17 00:00:00 2001 From: Philipp Schumacher Date: Thu, 13 Nov 2014 11:07:20 +0100 Subject: [PATCH 28/50] introduce configuration for intercepting by default --- .../greengerong/PreRenderSEOFilter.java | 2 +- .../github/greengerong/PrerenderConfig.java | 8 +++++++ .../greengerong/PrerenderSeoService.java | 21 +++++++++++-------- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index de582a1..c9545e1 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -13,7 +13,7 @@ public class PreRenderSEOFilter implements Filter { public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl"); + "blacklist", "prerenderServiceUrl", "interceptByDefault"); private PrerenderSeoService prerenderSeoService; @Override diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 16eeb6f..f1aa3d5 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -75,6 +75,14 @@ public String getPrerenderToken() { public String getForwardedURLHeader() { return config.get("forwardedURLHeader"); } + + public boolean isInterceptByDefault() { + String interceptByDefault = config.get("interceptByDefault"); + if(interceptByDefault != null) { + return Boolean.parseBoolean(interceptByDefault); + } + return true; + } public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 22445b9..958fc04 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -119,18 +119,21 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI return true; } - if (StringUtils.isBlank(userAgent)) { - log.trace("Request has blank userAgent; intercept: no"); - return false; - } + boolean interceptByDefault = prerenderConfig.isInterceptByDefault(); + if(interceptByDefault) { + if (StringUtils.isBlank(userAgent)) { + log.trace("Request has blank userAgent; intercept: no"); + return false; + } - if (!isInSearchUserAgent(userAgent)) { - log.trace("Request User-Agent is not a search bot; intercept: no"); - return false; + if (!isInSearchUserAgent(userAgent)) { + log.trace("Request User-Agent is not a search bot; intercept: no"); + return false; + } } - log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); - return true; + log.trace(String.format("Defaulting to request intercept(user-agent=%s): %s", userAgent, interceptByDefault ? "yes" : "no")); + return interceptByDefault; } protected HttpGet getHttpGet(String apiUrl) { From bd3c3f5740130dadac37153f79323dfa7c578737 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 13 Nov 2014 22:44:55 +0800 Subject: [PATCH 29/50] [pull request #29] revert change, let's check it first. --- .../greengerong/PreRenderSEOFilter.java | 2 +- .../github/greengerong/PrerenderConfig.java | 8 ------- .../greengerong/PrerenderSeoService.java | 21 ++++++++----------- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 4477a2d..6eb721f 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -14,7 +14,7 @@ public class PreRenderSEOFilter implements Filter { public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl", "interceptByDefault"); + "blacklist", "prerenderServiceUrl"); private PrerenderSeoService prerenderSeoService; @Override diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 55b6dc7..1219486 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -79,14 +79,6 @@ public String getPrerenderToken() { public String getForwardedURLHeader() { return config.get("forwardedURLHeader"); } - - public boolean isInterceptByDefault() { - String interceptByDefault = config.get("interceptByDefault"); - if(interceptByDefault != null) { - return Boolean.parseBoolean(interceptByDefault); - } - return true; - } public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 66ae08c..a59b697 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -124,21 +124,18 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI return true; } - boolean interceptByDefault = prerenderConfig.isInterceptByDefault(); - if(interceptByDefault) { - if (StringUtils.isBlank(userAgent)) { - log.trace("Request has blank userAgent; intercept: no"); - return false; - } + if (StringUtils.isBlank(userAgent)) { + log.trace("Request has blank userAgent; intercept: no"); + return false; + } - if (!isInSearchUserAgent(userAgent)) { - log.trace("Request User-Agent is not a search bot; intercept: no"); - return false; - } + if (!isInSearchUserAgent(userAgent)) { + log.trace("Request User-Agent is not a search bot; intercept: no"); + return false; } - log.trace(String.format("Defaulting to request intercept(user-agent=%s): %s", userAgent, interceptByDefault ? "yes" : "no")); - return interceptByDefault; + log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); + return true; } protected HttpGet getHttpGet(String apiUrl) { From 78a94b575a29b5ec3514e368785537dde7a512a4 Mon Sep 17 00:00:00 2001 From: greengerong Date: Fri, 14 Nov 2014 12:44:39 +0800 Subject: [PATCH 30/50] [pull request #29] remove google, bing and yahoo from defaults crawler. add new crawler sync "https://github.com/prerender/prerender-node/blob/master/index.js#L32" --- src/main/java/com/github/greengerong/PrerenderConfig.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 1219486..e647631 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -81,8 +81,9 @@ public String getForwardedURLHeader() { } public List getCrawlerUserAgents() { - List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", - "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly"); + List crawlerUserAgents = Lists.newArrayList("baiduspider", + "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview" + , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot"); final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); if (isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); From 3c02d915634ed19c43d8d6be1917d916f6ffae98 Mon Sep 17 00:00:00 2001 From: greengerong Date: Fri, 14 Nov 2014 21:13:19 +0800 Subject: [PATCH 31/50] [Tech] clean code. --- .../greengerong/PreRenderSEOFilterTest.java | 25 ++++++++++--------- .../greengerong/PrerenderConfigTest.java | 6 ++--- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 49229c2..6776586 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -23,13 +23,14 @@ import java.util.HashMap; import static com.github.greengerong.PrerenderSeoService.ESCAPED_FRAGMENT_KEY; +import static org.apache.http.HttpStatus.SC_NOT_FOUND; import static org.apache.http.HttpStatus.SC_OK; +import static org.apache.http.client.methods.HttpGet.METHOD_NAME; import static org.mockito.Mockito.*; @RunWith(MockitoJUnitRunner.class) public class PreRenderSEOFilterTest { - public static final int NOT_FOUND = 404; private PreRenderSEOFilter preRenderSEOFilter; @Mock @@ -95,7 +96,7 @@ public void should_handle_when_url_with_escaped_fragment_() throws Exception { final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); @@ -120,7 +121,7 @@ public void should_not_handle_when_user_agent_is_not_crawler() throws Exception preRenderSEOFilter.init(filterConfig); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); when(servletRequest.getHeader("User-Agent")).thenReturn("no"); //when @@ -138,7 +139,7 @@ public void should_not_handle_when_url_is_a_resource() throws Exception { preRenderSEOFilter.init(filterConfig); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test.js")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); //when @@ -157,7 +158,7 @@ public void should_not_handle_when_white_list_is_not_empty_and_url_is_not_in_whi preRenderSEOFilter.init(filterConfig); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); //when @@ -176,7 +177,7 @@ public void should_not_handle_when_black_list_is_not_empty_and_url_is_in_black_l preRenderSEOFilter.init(filterConfig); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); //when @@ -197,7 +198,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); @@ -227,7 +228,7 @@ public void should_handle_when_every_thing_is_ok_but_prerender_server_response_i final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); @@ -236,7 +237,7 @@ public void should_handle_when_every_thing_is_ok_but_prerender_server_response_i final HashMap map = Maps.newHashMap(); map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(NOT_FOUND); + when(statusLine.getStatusCode()).thenReturn(SC_NOT_FOUND); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); when(servletResponse.getWriter()).thenReturn(printWriter); @@ -247,7 +248,7 @@ public void should_handle_when_every_thing_is_ok_but_prerender_server_response_i //then verify(httpClient).execute(httpGet); verify(filterChain, never()).doFilter(servletRequest, servletResponse); - verify(servletResponse).setStatus(NOT_FOUND); + verify(servletResponse).setStatus(SC_NOT_FOUND); } @@ -264,7 +265,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); @@ -298,7 +299,7 @@ public void should_use_request_url_from_custom_header_if_available() throws Exce final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeader("X-Forwarded-URL")).thenReturn("http://my.public.domain.com/"); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); diff --git a/src/test/java/com/github/greengerong/PrerenderConfigTest.java b/src/test/java/com/github/greengerong/PrerenderConfigTest.java index d144e85..531c021 100644 --- a/src/test/java/com/github/greengerong/PrerenderConfigTest.java +++ b/src/test/java/com/github/greengerong/PrerenderConfigTest.java @@ -1,9 +1,9 @@ package com.github.greengerong; +import com.google.common.collect.Maps; import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Test; -import java.util.HashMap; import java.util.Map; import static org.hamcrest.core.Is.is; @@ -14,7 +14,7 @@ public class PrerenderConfigTest { @Test(expected = Exception.class) public void should_throw_exception_if_invalid_timeout_value_specified() throws Exception { //given - Map configuration = new HashMap(); + Map configuration = Maps.newHashMap(); configuration.put("socketTimeout", "not_an_int"); PrerenderConfig config = new PrerenderConfig(configuration); //when @@ -24,7 +24,7 @@ public void should_throw_exception_if_invalid_timeout_value_specified() throws E @Test public void should_pass_if_correct_timeout_value_specified() throws Exception { //given - Map configuration = new HashMap(); + Map configuration = Maps.newHashMap(); configuration.put("socketTimeout", "1000"); PrerenderConfig config = new PrerenderConfig(configuration); //when From 01d0511e859458078379b258972a066be58483c6 Mon Sep 17 00:00:00 2001 From: Geir Engdahl Date: Mon, 26 Jan 2015 14:23:39 +0100 Subject: [PATCH 32/50] Encode the response body with the encoding used by the proxy response. --- .../greengerong/PrerenderSeoService.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index a59b697..250b075 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -200,6 +200,7 @@ private String getApiUrl(String url) { * Copy proxied response headers back to the servlet client. */ private void copyResponseHeaders(HttpResponse proxyResponse, final HttpServletResponse servletResponse) { + servletResponse.setCharacterEncoding(getContentCharSet(proxyResponse.getEntity())); from(Arrays.asList(proxyResponse.getAllHeaders())).filter(new Predicate
() { @Override public boolean apply(Header header) { @@ -213,6 +214,26 @@ public Boolean apply(Header header) { } }).toList(); } + + /** + * Get the charset used to encode the http entity. + */ + private String getContentCharSet(final HttpEntity entity) throws ParseException { + if (entity == null) { + return null; + } + String charset = null; + if (entity.getContentType() != null) { + HeaderElement values[] = entity.getContentType().getElements(); + if (values.length > 0) { + NameValuePair param = values[0].getParameterByName("charset"); + if (param != null) { + charset = param.getValue(); + } + } + } + return charset; + } private String getResponseHtml(HttpResponse proxyResponse) throws IOException { From 83d571e0d872d032f863c3ccc19489a1e8104cb9 Mon Sep 17 00:00:00 2001 From: Florian Hirsch Date: Thu, 1 Oct 2015 09:27:53 +0700 Subject: [PATCH 33/50] #34: Disabling auto-redirect-handling --- src/main/java/com/github/greengerong/PrerenderConfig.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index e647631..1dc1598 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -42,7 +42,8 @@ public PreRenderEventHandler getEventHandler() { public CloseableHttpClient getHttpClient() { HttpClientBuilder builder = HttpClients.custom() - .setConnectionManager(new PoolingHttpClientConnectionManager()); + .setConnectionManager(new PoolingHttpClientConnectionManager()) + .disableRedirectHandling(); configureProxy(builder); configureTimeout(builder); From c5fda9faefac4fbdfdb3466e0427f5f9c49a5b19 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 8 Oct 2015 20:20:28 +0800 Subject: [PATCH 34/50] readme & update prerender client --- README.md | 25 +++++++++++-------- demo/pom.xml | 10 ++++---- demo/src/main/webapp/WEB-INF/web.xml | 8 +++--- prerender-java.iml | 3 +-- .../github/greengerong/PrerenderConfig.java | 19 ++++++++------ 5 files changed, 37 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 50d9bb9..1affbe8 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa 2. Check to make sure we aren't requesting a resource (js, css, etc...) 3. (optional) Check to make sure the url is in the whitelist 4. (optional) Check to make sure the url isn't in the blacklist -2. Make a `GET` request to the [prerender service](https://github.com/collectiveip/prerender)(phantomjs server) for the page's prerendered HTML +2. Make a `GET` request to the [prerender service](https://github.com/prerender/prerender)(phantomjs server) for the page's prerendered HTML 3. Return that HTML to the crawler ## Customization @@ -56,7 +56,7 @@ We usually set the original url in an http header which is added by the reverse ### Using your own prerender service -If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://prerender.herokuapp.com` +If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://service.prerender.io/` $ export PRERENDER_SERVICE_URL= @@ -92,16 +92,21 @@ If you want to cache the caching, analytics, log or others, you can config it. I ## Testing -If you want to make sure your pages are rendering correctly: +If your URLs use a hash-bang: -1. Open the Developer Tools in Chrome (Cmd + Atl + J) -2. Click the Settings gear in the bottom right corner. -3. Click "Overrides" on the left side of the settings panel. -4. Check the "User Agent" checkbox. -6. Choose "Other..." from the User Agent dropdown. -7. Type `googlebot` into the input box. -8. Refresh the page (make sure to keep the developer tools open). + If you want to see `http://localhost:3000/#!/profiles/1234` + Then go to `http://localhost:3000/?_escaped_fragment_=/profiles/1234` +If your URLs use push-state: + + If you want to see `http://localhost:3000/profiles/1234` + Then go to `http://localhost:3000/profiles/1234?_escaped_fragment_=` + + +project demo test url: + + http://localhost:8080/test/?_escaped_fragment_= + ## License The MIT License (MIT) diff --git a/demo/pom.xml b/demo/pom.xml index 2552382..076a7bb 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -13,11 +13,11 @@ prerender-java 1.6.2 - - com.github.greengerong - prerender-java-redis - 0.0.1-SNAPSHOT - + + + + + commons-io commons-io diff --git a/demo/src/main/webapp/WEB-INF/web.xml b/demo/src/main/webapp/WEB-INF/web.xml index c5e037b..2677fca 100644 --- a/demo/src/main/webapp/WEB-INF/web.xml +++ b/demo/src/main/webapp/WEB-INF/web.xml @@ -15,10 +15,10 @@ crawlerUserAgents YahooSeeker - - preRenderEventHandler - com.github.greengerong.PrerenderRedisCache - + + + + prerender diff --git a/prerender-java.iml b/prerender-java.iml index cbd3c69..f74aa01 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -23,5 +23,4 @@ - - + \ No newline at end of file diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 1dc1598..90c2240 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -2,7 +2,6 @@ import com.google.common.collect.Lists; -import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; @@ -84,7 +83,8 @@ public String getForwardedURLHeader() { public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview" - , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot"); + , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot", "vkShare", + "W3C_Validator", "redditbot", "Applebot"); final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); if (isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); @@ -94,10 +94,10 @@ public List getCrawlerUserAgents() { } public List getExtensionsToIgnore() { - List extensionsToIgnore = Lists.newArrayList(".xml", ".js", ".css", ".less", ".png", ".jpg", ".jpeg", - ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", - ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", - ".iso", ".flv", ".m4v", ".torrent"); + List extensionsToIgnore = Lists.newArrayList(".js", ".json", ".css", ".xml", ".less", ".png", ".jpg", + ".jpeg", ".gif", ".pdf", ".doc", ".txt", ".ico", ".rss", ".zip", ".mp3", ".rar", ".exe", ".wmv", + ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", + ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent", ".woff", ".ttf"); final String extensionsToIgnoreFromConfig = config.get("extensionsToIgnore"); if (isNotBlank(extensionsToIgnoreFromConfig)) { extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); @@ -124,6 +124,11 @@ public List getBlacklist() { public String getPrerenderServiceUrl() { final String prerenderServiceUrl = config.get("prerenderServiceUrl"); - return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : PRERENDER_IO_SERVICE_URL; + return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : getDefaultPrerenderIoServiceUrl(); + } + + private String getDefaultPrerenderIoServiceUrl() { + final String prerenderServiceUrlInEnv = System.getProperty("PRERENDER_SERVICE_URL"); + return isNotBlank(prerenderServiceUrlInEnv) ? prerenderServiceUrlInEnv : PRERENDER_IO_SERVICE_URL; } } From 99cefef83e1b17041dc9f7067c607b9f288fabe8 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 8 Oct 2015 20:27:55 +0800 Subject: [PATCH 35/50] [maven-release-plugin] prepare release prerender-java-1.6.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 112aab4..251e979 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.4-SNAPSHOT + 1.6.4 jar prerender-java From d7b7d943208d973f6dde62be243753a1df4cc99f Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 8 Oct 2015 20:28:19 +0800 Subject: [PATCH 36/50] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 251e979..01017d3 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.4 + 1.6.5-SNAPSHOT jar prerender-java From 29a1e08027ae926fe10d3627d027b98c8b717696 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 8 Oct 2015 20:50:54 +0800 Subject: [PATCH 37/50] new version 1.6.4 --- README.md | 2 +- demo/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1affbe8..7d0af15 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.6.4-SNAPSHOT + 1.6.4 ## How it works diff --git a/demo/pom.xml b/demo/pom.xml index 076a7bb..5b1c40c 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,7 +11,7 @@ com.github.greengerong prerender-java - 1.6.2 + 1.6.4 From 707e199407debb385cd2b0683549ccd099dc5a61 Mon Sep 17 00:00:00 2001 From: Jan Philip Bernius Date: Fri, 6 May 2016 20:03:02 +0200 Subject: [PATCH 38/50] Remove .DS_Store files --- .DS_Store | Bin 6148 -> 0 bytes .gitignore | 3 +++ demo/.DS_Store | Bin 6148 -> 0 bytes demo/src/.DS_Store | Bin 6148 -> 0 bytes demo/src/main/.DS_Store | Bin 6148 -> 0 bytes demo/src/main/webapp/.DS_Store | Bin 6148 -> 0 bytes src/main/.DS_Store | Bin 6148 -> 0 bytes src/main/java/.DS_Store | Bin 6148 -> 0 bytes src/main/java/com/.DS_Store | Bin 6148 -> 0 bytes src/main/java/com/github/.DS_Store | Bin 6148 -> 0 bytes 10 files changed, 3 insertions(+) delete mode 100644 .DS_Store delete mode 100644 demo/.DS_Store delete mode 100644 demo/src/.DS_Store delete mode 100644 demo/src/main/.DS_Store delete mode 100644 demo/src/main/webapp/.DS_Store delete mode 100644 src/main/.DS_Store delete mode 100644 src/main/java/.DS_Store delete mode 100644 src/main/java/com/.DS_Store delete mode 100644 src/main/java/com/github/.DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index f8b64f14cb1b4abba656ea103bb350b269a79956..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKQA+|r5S~@5J1X!c@Nwx`LM>CFheHf{3>19GsJl*5Fi*TQAIv&}eoz0V-_y+Q zDw*{lghb82>^D0*bIW{p?A!u?sE+(RKpFrnRKkRf!xuv9q)U=8o)n^>XL#dq;PnCz z=dVlA?D&lg(7Q{44{f-E;N$!A2EA?=<#KN#kxXrDZdtZ%?X;TRz8{RM!Km+t7xhjs zu6O;m=MFEG9l6mcthz(Ba_&YQTv2u-2zqF@OmwsBN1dy5W-=)j3t5@3RHj)uDV1_r zd30EwPHoGw(=XQE{=vyv!+n@OKE;|lfuEU@U4t`tMq^&aeSO&P1^rv}QG60Y7{bl^ ziz^h1Sln^>CyFygd@5~LJ|i>03@`($F<=f;YkRdcj%Eg!fq%dNoevU~&@mWmR7VF6 z>I#5JzmZDNrn>~?NP~{SSR<~W2%UCFheHf{3>19GsJl*75Kp`_AIv&}eoz0V-_y+Q z8kq$?ghb82>^D0*bIW{p?A!u?s7?F=Kn4ITRKkRf!xuv9q$`pzo)n^>XM`hf5O_Fy z@kT4r?D&lg(7Q`Q2xGYQrXSzmpdUv0{HsVLQ(Nh6%eJlEcB?=1gJ~_84E^x3(F>-H zzTffO@s+Y8H=2YscdS+}+^B~u%5Db10PVJkZuR}Bcb&=3X2nt|CkrQ)c}~vC<$O*a zA64da+p_G;v$cP4czWJ+@8=JXv1U%-N2X-Y-~yh|Fh$%o#=}7{?4pOF(-47+#dV5|kqiItF8nxPl^dDxyvm=87S7I{KxFa}35Bbvg*OGd{=cEX)l> zsNK;oRX7MoBah4gGqB7+Qcp9~|Ifa^|1T%;j2U1C{uBcuRd3X5Sd!_jE5%W-m8dtU pBor5Gd{03`wPK8=R-8puf__N`qGK@Dh#nOF5l}Snzzl4Zfj5k^fwBMq diff --git a/demo/src/.DS_Store b/demo/src/.DS_Store deleted file mode 100644 index e66c6b5437e4c9df30bd8a4979ba83365488deeb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~K}*9h6vvaTYKscJ2s;MPMK*PA2M?u8@F;Y!gA;A3xI%5m+PTB57QxTsxAF7% zUXqH_okipYlK)HIdr9+yEeT_cd-JHnn9Ue7p`vm$))X9G>uTs=RE~3mi+ma;SqS|% z?fdJ3-)^vb7O#ZfW2wp+{PVYlmu&Pl&?#Io0GJL34LUzV0> zTK22Cw|{VYJ`5hqr{_v>)`ghzk? diff --git a/demo/src/main/.DS_Store b/demo/src/main/.DS_Store deleted file mode 100644 index 496035a1af584566e79dc9088a9c3c0ee69cbcd5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z-NTO;8~hp~uB@kyfqk!9yq&JPIlFU_}#BY#=0|NzI{FgW&V{Ha?Ft zyPHEWXAwICv)}Ce>}Eg6{xHV)c$NivHzj+ZCJPu-6qA=cDb` zGz`=FFpf`7uWqL9%l7rHQ2cSATFQ~e65he6Eall*MPVF0fxoONa|ww7Vt^PR26mVM zeJB_QJInzsofsen>KVZOL4YFK8ViMT>wpHY&lqnZqJWKW2}GgM)>tTn2nbiDfU1=1 zCk9vL;1@d2)>tT1<&4XjVH`bk`FP=ScJK?C&bY0RT4I10s4~!0T?^0uYxreqANkc1 z>JbCPz&~SvwyPE(St~#vK~XTTKm!Eq$|V3gxQ}#XP{jq>kY{Tw6yhl8 Rm*s$T5s-vXLk#=^1K(ORNofE8 diff --git a/demo/src/main/webapp/.DS_Store b/demo/src/main/webapp/.DS_Store deleted file mode 100644 index 2a74ced11ee55ebb91e09228cecc47288de92ca0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5T3Qwrl^pM(BtB{N~_lP;30%o^ddqFJy_9%hz*3MG^st*Y7l%L-^S;0 zW_Jr#>s7?g!0tCYKf9R^vOfSI+;LC^umQlrMkq*GB4l3bS~J0fLd|gw7jWuNlA#|h zndmQ?=-Ug>g%-H52cNz__R;g^52G+iE0uS#mdmei6s%&=+Uj+~K@d;ecsvM_t9C!0 zw8P-e_eR%BPrY=UxZX&4F1@sm9;J8UI6}Q=qPt;`_HXRcbb3&$m1Xs~F)PbyyyvNzobEX*-NVbw~gL8O6LHlM(7yKG^(uwJGwrTKTk-4I=v+br9sDFrV%|T zLZ>3?RAHVNLZ_o$nmET`rctMZP%Gm+X65$=PMPZV!OpgLU9=>D5x)70&qb4NJ%-hUy_b- Yj=@YL%|dpSj>s1QMF@Ayz%MZH0p1};X#fBK diff --git a/src/main/.DS_Store b/src/main/.DS_Store deleted file mode 100644 index 1310f12577816836235239bd947be7ffa0766e36..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~K}*9h6vvaTYKscJ2s;MPLpF782M?u8@F;Y!gA;A3xJqfq+HHreS_D6j-^S15 zdr2xzcNUQsNd7N*?V;Vgo?`1*idkIqpPNaQ8~^LEV4-urvdce zwC}GAe!IaQna}3T|N8yGB#yGY)q3Z(y0NvrV_KHEHyTDsm@eFOmW0{0H%=E`6y68^ z^hVmbpU*PapGwb_pO3*K?LnHxV2^b4FbebWt=(8Ik2{^FXrJ|prdW2nt)@6V=@o@# znwI@)9vmK>Uk?0L@$_6OjvO${vahw07cf4zG54lPoF)%2R>fCwq(p!S5CI}U1ggWq z+$9!7fCvzQZv^o2D4?P`T1%z!UI`6epV4oiVgVc95~xC}qqS5D9w=O;f~r)mpBP-F z!?@6Kj@DACN@rZI4D;xftH%qMtHZdE;fy;yPc>Su3F4LB)b`B^pq`Ub_T92mePJ(y9CcZHRNUmP&CJ9GB^! ObPSjXs2s=0rePvOsJ(|% zP`Ha52`+qQPbqd1M^!jtnf$}athT=;Qx7Y4H{ zW#>UY&%9u!JQqPe0gtkWQYK)JP4p;^^2xQ+TrLm0-InN_^oy2Q_ImA>I6mqZg>6~3 z^JeYsADo^KgQw#8rPLHTKvrqjXeF;;m_hZ$W?9eV$IODEXDv1CQSS3)`RR_=iGx%lJKJr()P>l!> zfqzDTHwON|gGHIM^~dt?tQC-NAY;O~5)DXTuUrD4g8N8QIkjJ)4RNl~QY(&v?W#K{ OT?7;mDu}=@5cmYR6Gp25 diff --git a/src/main/java/com/.DS_Store b/src/main/java/com/.DS_Store deleted file mode 100644 index e0512af466f9b9e96547f2cda22eb0628ed9b252..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z-NTO;8~hp~uB@kyfqk!9xfYJPIlFU_}!mO`tTTN$jCUgW&V{Ha?Ft zyPHEWXAwICv)}Ce>}Eg6{xHV4lLdXoEXJ4tipbHZ5p>sv8af$~%P}(M`}Fy>FPNaX6bs%OKu3akdPSJ2#9sZWKKE z-uhnJsh4Jn@n)=NW(NP7~;3$SN8dm064_+WLm+q1!-EBcqCyeqcDVXrGL&PRD} z8ir|o7{@24S2q*yC4YS@6n`A3mU5)Agm*A1OL=zJ%SF7LgTJgPa|ww7Vt^PR26mVM zeJB_QJInzsofsen>KVZOL4YFK8ViMT>wpHY&lqnZqJWKW2}GgM)>tTn2nbiDfU1=1 zCk9vL;1@d2)>tT1<&4XjVH`bk`FP=ScJK?C&bY0RT4I10s4~!0T?^0uYxreqANkc1 z>JbCPz&~Svx5n<+fko-F^~dt?tQDY-peUGEpaBAQ%6b|%Zb3AY~)C*$|BHp}d7^GRV`OZt_%FgbdVVcJNa1ca( zJh$Un7VzV$ta|8{+J=o9Cusn6+2q@aXvBs_#DLPtS#7lEX(Qr5V#Ntt0 b2bBVTfd-(Zu}}ye5c(02G|)f{{3-( Date: Sun, 29 Apr 2018 01:19:14 +0530 Subject: [PATCH 39/50] #44 added provision for specifying protocol --- README.md | 3 +++ .../greengerong/PreRenderSEOFilter.java | 19 +++++++++++-------- .../github/greengerong/PrerenderConfig.java | 4 ++++ .../greengerong/PrerenderSeoService.java | 8 ++++++-- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7d0af15..f2af0fc 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,9 @@ example: someproxy,someproxy1 Important for servers behind reverse proxy that need the public url to be used for pre-rendering. We usually set the original url in an http header which is added by the reverse proxy (similar to the more standard `x-forwarded-proto` and `x-forwarded-for`) +### protocol +If you specifically want to make sure that the Prerender service queries using http or https protocol, you can set the init-param `protocol` to `https` or `http` respectively. + ### Using your own prerender service If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://service.prerender.io/` diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 6eb721f..39195f9 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,20 +1,23 @@ package com.github.greengerong; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -import javax.servlet.*; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; public class PreRenderSEOFilter implements Filter { public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl"); + "blacklist", "prerenderServiceUrl", "protocol"); private PrerenderSeoService prerenderSeoService; @Override diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 90c2240..a18d314 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -79,6 +79,10 @@ public String getPrerenderToken() { public String getForwardedURLHeader() { return config.get("forwardedURLHeader"); } + + public String getProtocol() { + return config.get("protocol"); + } public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("baiduspider", diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 250b075..4ab980e 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -179,12 +179,16 @@ private void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest p } private String getRequestURL(HttpServletRequest request) { - if (prerenderConfig.getForwardedURLHeader() != null) { + if (StringUtils.isNotEmpty(prerenderConfig.getForwardedURLHeader())) { String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); - if (url != null) { + if (StringUtils.isNotEmpty(url)) { return url; } } + if (StringUtils.isNotEmpty(prerenderConfig.getProtocol())) { + String url = request.getRequestURL().toString(); + return url.replace(request.getScheme(), prerenderConfig.getProtocol()); + } return request.getRequestURL().toString(); } From 4f5c76f29e0c1fb63dcd31744bc27da1ca622e7a Mon Sep 17 00:00:00 2001 From: Karan Gujral Date: Sun, 29 Apr 2018 01:23:33 +0530 Subject: [PATCH 40/50] #44 changed readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f2af0fc..0486019 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Important for servers behind reverse proxy that need the public url to be used f We usually set the original url in an http header which is added by the reverse proxy (similar to the more standard `x-forwarded-proto` and `x-forwarded-for`) ### protocol -If you specifically want to make sure that the Prerender service queries using http or https protocol, you can set the init-param `protocol` to `https` or `http` respectively. +If you specifically want to make sure that the Prerender service queries using https or http protocol, you can set the init-param `protocol` to `https` or `http` respectively. ### Using your own prerender service From 049355625b8d074209511987a965538bbd2b082d Mon Sep 17 00:00:00 2001 From: green Date: Mon, 7 May 2018 11:11:19 +0800 Subject: [PATCH 41/50] Update .travis.yml --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index aec209e..e8f05af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: java script: "mvn clean install -U" jdk: - #- oraclejdk6 - - oraclejdk7 - - openjdk6 - - openjdk7 \ No newline at end of file + - openjdk7 + - openjdk8 From 23b6fd6370ced4c38325405c8019e26bf36c6557 Mon Sep 17 00:00:00 2001 From: Karan Gujral Date: Fri, 11 May 2018 14:04:10 -0500 Subject: [PATCH 42/50] trying to re-run CI --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0486019..edf8c2e 100644 --- a/README.md +++ b/README.md @@ -112,4 +112,4 @@ project demo test url: ## License -The MIT License (MIT) +The MIT License (MIT) \ No newline at end of file From b2583beb0a34c0566d7ddf80ce38e767ce7d6776 Mon Sep 17 00:00:00 2001 From: green Date: Wed, 1 Aug 2018 11:38:08 +0800 Subject: [PATCH 43/50] LICENSE --- LICENSE.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..43db8ab --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017-present, GreenGerong(https://github.com/greengerong), Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From d677fb0e3a99f59bd76995e596688d13c1504b3d Mon Sep 17 00:00:00 2001 From: Nathan McWilliams Date: Thu, 23 Aug 2018 17:17:37 -0400 Subject: [PATCH 44/50] change logging of final request url from log.trace to log.info --- src/main/java/com/github/greengerong/PrerenderSeoService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 4ab980e..288e869 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -331,7 +331,7 @@ private boolean beforeRender(HttpServletRequest request, HttpServletResponse res private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException { final String apiUrl = getApiUrl(getFullUrl(request)); - log.trace(String.format("Prerender proxy will send request to:%s", apiUrl)); + log.info(String.format("Prerender proxy will send request to:%s", apiUrl)); final HttpGet getMethod = getHttpGet(apiUrl); copyRequestHeaders(request, getMethod); withPrerenderToken(getMethod); From f01e29c4c0d590a380143f3b83f75e9a3702478e Mon Sep 17 00:00:00 2001 From: Todd Hooper Date: Mon, 15 Oct 2018 20:51:37 -0500 Subject: [PATCH 45/50] Add a check for our X-Prerender header Make sure requests from the prerender browser don't get sent through to Prerender.io --- .../java/com/github/greengerong/PrerenderSeoService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 4ab980e..043a4a9 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -106,6 +106,11 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI log.trace("request is for a (static) resource; intercept: no"); return false; } + + String prerenderHeader = request.getHeader("X-Prerender"); + if (StringUtils.isNotEmpty(prerenderHeader)) { + return false; + } final List whiteList = prerenderConfig.getWhitelist(); if (whiteList != null && !isInWhiteList(url, whiteList)) { From df2d5438c97ee6f55003ff3337375a27642c5a4a Mon Sep 17 00:00:00 2001 From: Todd Hooper Date: Mon, 15 Oct 2018 20:58:55 -0500 Subject: [PATCH 46/50] added a test for the X-Prerender header check --- .../greengerong/PreRenderSEOFilterTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 6776586..ba81d67 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -132,6 +132,25 @@ public void should_not_handle_when_user_agent_is_not_crawler() throws Exception verify(filterChain).doFilter(servletRequest, servletResponse); } + @Test + public void should_not_handle_when_x_prerender_header_is_present() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + when(servletRequest.getHeader("X-Prerender")).thenReturn("1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + @Test public void should_not_handle_when_url_is_a_resource() throws Exception { //given From 09a90437c5e848956842a449698cda13254c5c45 Mon Sep 17 00:00:00 2001 From: Csaba Varro Date: Wed, 24 Nov 2021 13:58:45 +0100 Subject: [PATCH 47/50] Extend user agent list --- src/main/java/com/github/greengerong/PrerenderConfig.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index a18d314..6b4e894 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -79,7 +79,7 @@ public String getPrerenderToken() { public String getForwardedURLHeader() { return config.get("forwardedURLHeader"); } - + public String getProtocol() { return config.get("protocol"); } @@ -88,7 +88,9 @@ public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview" , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot", "vkShare", - "W3C_Validator", "redditbot", "Applebot"); + "W3C_Validator", "redditbot", "Applebot", "whatsapp", "flipboard", "tumblr", "bitlybot", + "skypeuripreview", "nuzzel", "discordbot", "google page speed", "qwantify", "pinterestbot", + "bitrix link preview", "xing-contenttabreceiver", "chrome-lighthouse", "telegrambot"); final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); if (isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); From 6c415e843ce143e2eab3a21b27b00b143d68ba0b Mon Sep 17 00:00:00 2001 From: greengerong Date: Mon, 12 Jun 2023 00:38:18 +0800 Subject: [PATCH 48/50] rename word --- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 39195f9..1fa4719 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -28,9 +28,9 @@ public void init(FilterConfig filterConfig) throws ServletException { @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - boolean isPrerendered = prerenderSeoService.prerenderIfEligible( + boolean isPreRendered = prerenderSeoService.prerenderIfEligible( (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); - if (!isPrerendered) { + if (!isPreRendered) { filterChain.doFilter(servletRequest, servletResponse); } } From 17618ffac877c9e0aa755f18c6bfb1ac103da8a3 Mon Sep 17 00:00:00 2001 From: green Date: Sat, 17 Jun 2023 11:41:14 +0800 Subject: [PATCH 49/50] Update PreRenderSEOFilter.java socketTimeout --- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 1fa4719..5704025 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -17,7 +17,7 @@ public class PreRenderSEOFilter implements Filter { public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl", "protocol"); + "blacklist", "prerenderServiceUrl", "protocol","socketTimeout"); private PrerenderSeoService prerenderSeoService; @Override From 292bd36335126dd9ea3fd8329ce629efddeeb91c Mon Sep 17 00:00:00 2001 From: green Date: Sun, 9 Jul 2023 16:28:03 +0800 Subject: [PATCH 50/50] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index edf8c2e..4d3f00f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Prerender Java [![Build Status](https://travis-ci.org/greengerong/prerender-java.png)](https://travis-ci.org/greengerong/prerender-java) +Prerender Java =========================== Are you using backbone, angular, emberjs, etc, but you're unsure about the SEO implications? @@ -112,4 +112,4 @@ project demo test url: ## License -The MIT License (MIT) \ No newline at end of file +The MIT License (MIT)