diff --git a/.gitignore b/.gitignore index 1508b92..112f2f2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,11 @@ .idea/ +.project +.classpath +.settings/ target/ **/target/ -**/.idea/ \ No newline at end of file +**/.idea/ +*.iml + +### OSX ### +.DS_Store diff --git a/.travis.yml b/.travis.yml index c2d9b47..e8f05af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: java script: "mvn clean install -U" jdk: - - oraclejdk6 - #- oraclejdk7 - - openjdk6 - - openjdk7 \ No newline at end of file + - openjdk7 + - openjdk8 diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..43db8ab --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017-present, GreenGerong(https://github.com/greengerong), Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 6ec66ca..4d3f00f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Prerender Java [![Build Status](https://travis-ci.org/greengerong/prerender-java.png)](https://travis-ci.org/greengerong/prerender-java) +Prerender Java =========================== Are you using backbone, angular, emberjs, etc, but you're unsure about the SEO implications? @@ -15,12 +15,8 @@ Use this java filter that prerenders a javascript-rendered page using an externa prerender com.github.greengerong.PreRenderSEOFilter - prerenderServiceUrl - http://localhost:3000 - - - crawlerUserAgents - me + prerenderToken + [get from prerender: https://prerender.io/] @@ -33,7 +29,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.0 + 1.6.4 ## How it works @@ -42,7 +38,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa 2. Check to make sure we aren't requesting a resource (js, css, etc...) 3. (optional) Check to make sure the url is in the whitelist 4. (optional) Check to make sure the url isn't in the blacklist -2. Make a `GET` request to the [prerender service](https://github.com/collectiveip/prerender)(phantomjs server) for the page's prerendered HTML +2. Make a `GET` request to the [prerender service](https://github.com/prerender/prerender)(phantomjs server) for the page's prerendered HTML 3. Return that HTML to the crawler ## Customization @@ -54,10 +50,16 @@ example: someproxy,someproxy1 ### blacklist +### forwardedURLHeader +Important for servers behind reverse proxy that need the public url to be used for pre-rendering. +We usually set the original url in an http header which is added by the reverse proxy (similar to the more standard `x-forwarded-proto` and `x-forwarded-for`) + +### protocol +If you specifically want to make sure that the Prerender service queries using https or http protocol, you can set the init-param `protocol` to `https` or `http` respectively. ### Using your own prerender service -If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://prerender.herokuapp.com` +If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://service.prerender.io/` $ export PRERENDER_SERVICE_URL= @@ -71,18 +73,43 @@ As an alternative, you can pass `prerender_service_url` in the options object du config filter init param with "prerenderServiceUrl"; ``` +### prerender service token + +If you want to use token with the prerender service, you can config it. + + +``` xml + config filter init param with "prerenderToken"; +``` + + +### prerender event handler + +If you want to cache the caching, analytics, log or others, you can config it. It should be instance of "com.github.greengerong.PreRenderEventHandler" + + +``` xml + config filter init param with "preRenderEventHandler"; +``` + + ## Testing -If you want to make sure your pages are rendering correctly: +If your URLs use a hash-bang: + + If you want to see `http://localhost:3000/#!/profiles/1234` + Then go to `http://localhost:3000/?_escaped_fragment_=/profiles/1234` + +If your URLs use push-state: + + If you want to see `http://localhost:3000/profiles/1234` + Then go to `http://localhost:3000/profiles/1234?_escaped_fragment_=` + -1. Open the Developer Tools in Chrome (Cmd + Atl + J) -2. Click the Settings gear in the bottom right corner. -3. Click "Overrides" on the left side of the settings panel. -4. Check the "User Agent" checkbox. -6. Choose "Other..." from the User Agent dropdown. -7. Type `googlebot` into the input box. -8. Refresh the page (make sure to keep the developer tools open). +project demo test url: + http://localhost:8080/test/?_escaped_fragment_= + ## License The MIT License (MIT) diff --git a/demo/demo.iml b/demo/demo.iml index c89231a..5215d4d 100644 --- a/demo/demo.iml +++ b/demo/demo.iml @@ -16,17 +16,18 @@ - - - + + - - - + + + + + diff --git a/demo/pom.xml b/demo/pom.xml index 6864939..5b1c40c 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -4,17 +4,75 @@ com.github.greengerong demo war - 1.0 + 1.0-SNAPSHOT demo Maven Webapp http://maven.apache.org com.github.greengerong prerender-java - 1.0-SNAPSHOT + 1.6.4 + + + + + + + + commons-io + commons-io + 2.4 + + + org.slf4j + slf4j-log4j12 + 1.7.5 + + + log4j + log4j + 1.2.17 demo + + + + org.mortbay.jetty + maven-jetty-plugin + 6.1.10 + + + + /test + ${project.build.directory}/demo/ + + 10 + foo + 8080 + + + + start-jetty + pre-integration-test + + run + + + 0 + true + + + + stop-jetty + post-integration-test + + stop + + + + + diff --git a/demo/src/main/resources/log4j.properties b/demo/src/main/resources/log4j.properties new file mode 100644 index 0000000..3074ab0 --- /dev/null +++ b/demo/src/main/resources/log4j.properties @@ -0,0 +1,63 @@ +#log4j.rootLogger=CONSOLE,FILE +log4j.rootLogger=DEBUG,CONSOLE +log4j.addivity.org.apache=true + + +# 应用于控制台 +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=INFO +log4j.appender.CONSOLE.Target=System.out +log4j.appender.CONSOLE.Encoding=GBK +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +# 每天新建日志 +log4j.appender.A1=org.apache.log4j.DailyRollingFileAppender +log4j.appender.A1.File=C:/log4j/log +log4j.appender.A1.Encoding=GBK +log4j.appender.A1.Threshold=DEBUG +log4j.appender.A1.DatePattern='.'yyyy-MM-dd +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L : %m%n + +#应用于文件 +log4j.appender.FILE=org.apache.log4j.FileAppender +log4j.appender.FILE.File=C:/log4j/file.log +log4j.appender.FILE.Append=false +log4j.appender.FILE.Encoding=GBK +log4j.appender.FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +# 应用于文件回滚 +log4j.appender.ROLLING_FILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLING_FILE.Threshold=ERROR +log4j.appender.ROLLING_FILE.File=rolling.log +log4j.appender.ROLLING_FILE.Append=true +log4j.appender.CONSOLE_FILE.Encoding=GBK +log4j.appender.ROLLING_FILE.MaxFileSize=10KB +log4j.appender.ROLLING_FILE.MaxBackupIndex=1 +log4j.appender.ROLLING_FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLING_FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +#应用于socket +log4j.appender.SOCKET=org.apache.log4j.RollingFileAppender +log4j.appender.SOCKET.RemoteHost=localhost +log4j.appender.SOCKET.Port=5001 +log4j.appender.SOCKET.LocationInfo=true +# Set up for Log Facter 5 +log4j.appender.SOCKET.layout=org.apache.log4j.PatternLayout +log4j.appender.SOCET.layout.ConversionPattern=[start]%d{DATE}[DATE]%n%p[PRIORITY]%n%x[NDC]%n%t[THREAD]%n%c[CATEGORY]%n%m[MESSAGE]%n%n +# Log Factor 5 Appender +log4j.appender.LF5_APPENDER=org.apache.log4j.lf5.LF5Appender +log4j.appender.LF5_APPENDER.MaxNumberOfRecords=2000 + +# 发送日志给邮件 +log4j.appender.MAIL=org.apache.log4j.net.SMTPAppender +log4j.appender.MAIL.Threshold=FATAL +log4j.appender.MAIL.BufferSize=10 +log4j.appender.MAIL.From= +log4j.appender.MAIL.SMTPHost=www.wusetu.com +log4j.appender.MAIL.Subject=Log4J Message +log4j.appender.MAIL.To= +log4j.appender.MAIL.layout=org.apache.log4j.PatternLayout +log4j.appender.MAIL.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n diff --git a/demo/src/main/webapp/WEB-INF/web.xml b/demo/src/main/webapp/WEB-INF/web.xml index 89ebe63..2677fca 100644 --- a/demo/src/main/webapp/WEB-INF/web.xml +++ b/demo/src/main/webapp/WEB-INF/web.xml @@ -13,8 +13,12 @@ crawlerUserAgents - me + YahooSeeker + + + + prerender diff --git a/demo/src/main/webapp/index.jsp b/demo/src/main/webapp/index.jsp index b1e3b13..4a09353 100644 --- a/demo/src/main/webapp/index.jsp +++ b/demo/src/main/webapp/index.jsp @@ -1,176 +1,18 @@ - - - - -

Prerender java demo page

- -

use angularjs ng-repeat

-
    -
  • {{item}}
    • -
- - + + + + + +

Prerender java demo page

+

use angularjs ng-repeat

+
    +
  • {{item}}
    • +
+ + diff --git a/pom.xml b/pom.xml index f506089..01017d3 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.1-SNAPSHOT + 1.6.5-SNAPSHOT jar prerender-java @@ -15,6 +15,7 @@ http://www.apache.org/licenses/LICENSE-2.0.txt repo + scm:git:git@github.com:greengerong/prerender-java.git @@ -32,23 +33,19 @@ 4.11 1.9.5 15.0 - 2.6 + 2.6 + 2.4 2.5 - 3.1 - 1.2.9 + 4.3.1 + 1.7.5 - - junit - junit - 4.11 - test - com.google.guava guava ${guava.version} + cdi1.0 junit @@ -65,7 +62,12 @@ commons-lang commons-lang - ${commons-lange.version} + ${commons-lang.version} + + + commons-io + commons-io + ${commons-io.version} javax.servlet @@ -74,27 +76,27 @@ provided - commons-httpclient - commons-httpclient - ${commons-httpclient.version} + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + org.slf4j + slf4j-api + ${slf4j.version} - - - - - sonatype-nexus-snapshots Sonatype Nexus Snapshots - http://oss.sonatype.org/content/repositories/snapshots + https://oss.sonatype.org/content/repositories/snapshots sonatype-nexus-staging Nexus Release Repository - http://oss.sonatype.org/service/local/staging/deploy/maven2/ + https://oss.sonatype.org/service/local/staging/deploy/maven2/ @@ -113,9 +115,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.7 - - + 2.8.1 attach-javadoc diff --git a/prerender-java.iml b/prerender-java.iml index 2dc2562..f74aa01 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -10,15 +10,17 @@ + - + - - - + + + + + - - + \ No newline at end of file diff --git a/src/.DS_Store b/src/.DS_Store new file mode 100644 index 0000000..e66c6b5 Binary files /dev/null and b/src/.DS_Store differ diff --git a/src/main/java/com/github/greengerong/PreRenderEventHandler.java b/src/main/java/com/github/greengerong/PreRenderEventHandler.java new file mode 100644 index 0000000..bd71634 --- /dev/null +++ b/src/main/java/com/github/greengerong/PreRenderEventHandler.java @@ -0,0 +1,16 @@ +package com.github.greengerong; + + +import org.apache.http.HttpResponse; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +public interface PreRenderEventHandler { + + String beforeRender(HttpServletRequest clientRequest); + + String afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, HttpResponse prerenderResponse, String responseHtml); + + void destroy(); +} diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 5e9defd..5704025 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,237 +1,55 @@ package com.github.greengerong; - -import com.google.common.base.Predicate; -import com.google.common.collect.Lists; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpMethod; -import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.commons.lang.StringUtils; - -import javax.servlet.*; -import javax.servlet.http.HttpServletRequest; import java.io.IOException; -import java.io.PrintWriter; -import java.net.URISyntaxException; -import java.util.Arrays; import java.util.List; -import java.util.regex.Pattern; - -import static com.google.common.collect.FluentIterable.from; +import java.util.Map; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; public class PreRenderSEOFilter implements Filter { - - private FilterConfig filterConfig; + public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", + "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", + "blacklist", "prerenderServiceUrl", "protocol","socketTimeout"); + private PrerenderSeoService prerenderSeoService; @Override public void init(FilterConfig filterConfig) throws ServletException { - this.filterConfig = filterConfig; + this.prerenderSeoService = new PrerenderSeoService(toMap(filterConfig)); } @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - try { - final HttpServletRequest request = (HttpServletRequest) servletRequest; - if (shouldShowPrerenderedPage(request)) { - final ResponseResult result = getPrerenderedPageResponse(request); - if (result.getStatusCode() == 200) { - final PrintWriter writer = servletResponse.getWriter(); - writer.write(result.getResponseBody()); - writer.flush(); - return; - } - } - } catch (Exception e) { - } - filterChain.doFilter(servletRequest, servletResponse); - } - - private ResponseResult getPrerenderedPageResponse(HttpServletRequest request) throws IOException { - final String apiUrl = getApiUrl(getFullUrl(request)); - final HttpClient httpClient = new HttpClient(); - final GetMethod getMethod = new GetMethod(apiUrl); - setConfig(httpClient); - setHttpHeader(getMethod); - final int code = httpClient.executeMethod(getMethod); - final String responseBody = getMethod.getResponseBodyAsString(); - String body = new String(responseBody.getBytes("utf-8")); - return new ResponseResult(code, body); - } - - private String getFullUrl(HttpServletRequest request) { - final StringBuffer url = request.getRequestURL(); - final String queryString = request.getQueryString(); - if (queryString != null) { - url.append('?'); - url.append(queryString); - } - return url.toString(); - } - - private void setHttpHeader(HttpMethod httpMethod) { - httpMethod.setRequestHeader("Cache-Control", "no-cache"); - httpMethod.setRequestHeader("Content-Type", "text/html"); - } - - private void setConfig(HttpClient httpClient) { - final String proxy = filterConfig.getInitParameter("proxy"); - if (StringUtils.isNotBlank(proxy)) { - final int proxyPort = Integer.parseInt(filterConfig.getInitParameter("proxyPort")); - httpClient.getHostConfiguration().setProxy(proxy, proxyPort); + boolean isPreRendered = prerenderSeoService.prerenderIfEligible( + (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); + if (!isPreRendered) { + filterChain.doFilter(servletRequest, servletResponse); } } @Override public void destroy() { - filterConfig = null; + prerenderSeoService.destroy(); } - private List getCrawlerUserAgents() { - List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", - "facebookexternalhit", "twitterbot"); - final String crawlerUserAgentsFromConfig = filterConfig.getInitParameter("crawlerUserAgents"); - if (StringUtils.isNotBlank(crawlerUserAgentsFromConfig)) { - crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); - } - - return crawlerUserAgents; + protected void setPrerenderSeoService(PrerenderSeoService prerenderSeoService) { + this.prerenderSeoService = prerenderSeoService; } - private List getExtensionsToIgnore() { - List extensionsToIgnore = Lists.newArrayList(".js", ".css", ".less", ".png", ".jpg", ".jpeg", - ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", - ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", - ".iso", ".flv", ".m4v", ".torrent"); - final String extensionsToIgnoreFromConfig = filterConfig.getInitParameter("extensionsToIgnore"); - if (StringUtils.isNotBlank(extensionsToIgnoreFromConfig)) { - extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); - } - - return extensionsToIgnore; - } - - private List getWhitelist() { - final String whitelist = filterConfig.getInitParameter("whitelist"); - if (StringUtils.isNotBlank(whitelist)) { - return Arrays.asList(whitelist.trim().split(",")); - } - return null; - } - - private List getBlacklist() { - final String blacklist = filterConfig.getInitParameter("blacklist"); - if (StringUtils.isNotBlank(blacklist)) { - return Arrays.asList(blacklist.trim().split(",")); - } - return null; - } - - private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { - final String useAgent = request.getHeader("User-Agent"); - final String url = request.getRequestURL().toString(); - final String referer = request.getHeader("Referer"); - - if (hasEscapedFragment(request)) { - return true; - } - - if (StringUtils.isBlank(useAgent)) { - return false; - } - - if (!isInSearchUserAgent(useAgent)) { - return false; - } - - - if (isInResources(url)) { - return false; - } - - final List whiteList = getWhitelist(); - if (whiteList != null && !isInWhiteList(url, whiteList)) { - return false; - } - - final List blacklist = getBlacklist(); - if (blacklist != null && isInBlackList(url, referer, blacklist)) { - return false; - } - - return true; - } - - private boolean hasEscapedFragment(HttpServletRequest request) { - return request.getParameterMap().containsKey("_escaped_fragment_"); - } - - private String getApiUrl(String url) { - String prerenderServiceUrl = getPrerenderServiceUrl(); - if (!prerenderServiceUrl.endsWith("/")) { - prerenderServiceUrl += "/"; - } - return prerenderServiceUrl + url; - } - - private String getPrerenderServiceUrl() { - final String prerenderServiceUrl = filterConfig.getInitParameter("prerenderServiceUrl"); - return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : "http://prerender.herokuapp.com/"; - } - - private boolean isInBlackList(final String url, final String referer, List blacklist) { - return from(blacklist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - final Pattern pattern = Pattern.compile(regex); - return pattern.matcher(url).matches() || - (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); - } - }); - } - - private boolean isInWhiteList(final String url, List whitelist) { - return from(whitelist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - return Pattern.compile(regex).matcher(url).matches(); - } - }); - } - - private boolean isInResources(final String url) { - return from(getExtensionsToIgnore()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return url.contains(item.toLowerCase()); - } - }); - } - - private boolean isInSearchUserAgent(final String useAgent) { - return from(getCrawlerUserAgents()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return item.equalsIgnoreCase(useAgent); - } - }); - } - - private class ResponseResult { - private int statusCode; - private String responseBody; - - public ResponseResult(int code, String body) { - statusCode = code; - responseBody = body; - } - - private int getStatusCode() { - return statusCode; - } - - private String getResponseBody() { - return responseBody; + protected Map toMap(FilterConfig filterConfig) { + Map config = Maps.newHashMap(); + for (String parameterName : PARAMETER_NAMES) { + config.put(parameterName, filterConfig.getInitParameter(parameterName)); } + return config; } } + diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java new file mode 100644 index 0000000..6b4e894 --- /dev/null +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -0,0 +1,140 @@ +package com.github.greengerong; + + +import com.google.common.collect.Lists; +import org.apache.http.HttpHost; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.DefaultProxyRoutePlanner; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.apache.commons.lang.StringUtils.isNotBlank; + +public class PrerenderConfig { + private final static Logger log = LoggerFactory.getLogger(PrerenderConfig.class); + public static final String PRERENDER_IO_SERVICE_URL = "http://service.prerender.io/"; + private Map config; + + public PrerenderConfig(Map config) { + this.config = config; + } + + public PreRenderEventHandler getEventHandler() { + final String preRenderEventHandler = config.get("preRenderEventHandler"); + if (isNotBlank(preRenderEventHandler)) { + try { + return (PreRenderEventHandler) Class.forName(preRenderEventHandler).newInstance(); + } catch (Exception e) { + log.error("PreRenderEventHandler class not find or can not new a instance", e); + } + } + return null; + } + + public CloseableHttpClient getHttpClient() { + HttpClientBuilder builder = HttpClients.custom() + .setConnectionManager(new PoolingHttpClientConnectionManager()) + .disableRedirectHandling(); + + configureProxy(builder); + configureTimeout(builder); + return builder.build(); + } + + private HttpClientBuilder configureProxy(HttpClientBuilder builder) { + final String proxy = config.get("proxy"); + if (isNotBlank(proxy)) { + final int proxyPort = Integer.parseInt(config.get("proxyPort")); + DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); + builder.setRoutePlanner(routePlanner); + } + return builder; + } + + private HttpClientBuilder configureTimeout(HttpClientBuilder builder) { + final String socketTimeout = getSocketTimeout(); + if (socketTimeout != null) { + RequestConfig config = RequestConfig.custom().setSocketTimeout(Integer.parseInt(socketTimeout)).build(); + builder.setDefaultRequestConfig(config); + } + return builder; + } + + public String getSocketTimeout() { + return config.get("socketTimeout"); + } + + public String getPrerenderToken() { + return config.get("prerenderToken"); + } + + public String getForwardedURLHeader() { + return config.get("forwardedURLHeader"); + } + + public String getProtocol() { + return config.get("protocol"); + } + + public List getCrawlerUserAgents() { + List crawlerUserAgents = Lists.newArrayList("baiduspider", + "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview" + , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot", "vkShare", + "W3C_Validator", "redditbot", "Applebot", "whatsapp", "flipboard", "tumblr", "bitlybot", + "skypeuripreview", "nuzzel", "discordbot", "google page speed", "qwantify", "pinterestbot", + "bitrix link preview", "xing-contenttabreceiver", "chrome-lighthouse", "telegrambot"); + final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); + if (isNotBlank(crawlerUserAgentsFromConfig)) { + crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); + } + + return crawlerUserAgents; + } + + public List getExtensionsToIgnore() { + List extensionsToIgnore = Lists.newArrayList(".js", ".json", ".css", ".xml", ".less", ".png", ".jpg", + ".jpeg", ".gif", ".pdf", ".doc", ".txt", ".ico", ".rss", ".zip", ".mp3", ".rar", ".exe", ".wmv", + ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", + ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent", ".woff", ".ttf"); + final String extensionsToIgnoreFromConfig = config.get("extensionsToIgnore"); + if (isNotBlank(extensionsToIgnoreFromConfig)) { + extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); + } + + return extensionsToIgnore; + } + + public List getWhitelist() { + final String whitelist = config.get("whitelist"); + if (isNotBlank(whitelist)) { + return Arrays.asList(whitelist.trim().split(",")); + } + return null; + } + + public List getBlacklist() { + final String blacklist = config.get("blacklist"); + if (isNotBlank(blacklist)) { + return Arrays.asList(blacklist.trim().split(",")); + } + return null; + } + + public String getPrerenderServiceUrl() { + final String prerenderServiceUrl = config.get("prerenderServiceUrl"); + return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : getDefaultPrerenderIoServiceUrl(); + } + + private String getDefaultPrerenderIoServiceUrl() { + final String prerenderServiceUrlInEnv = System.getProperty("PRERENDER_SERVICE_URL"); + return isNotBlank(prerenderServiceUrlInEnv) ? prerenderServiceUrlInEnv : PRERENDER_IO_SERVICE_URL; + } +} diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java new file mode 100644 index 0000000..63d3352 --- /dev/null +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -0,0 +1,378 @@ +package com.github.greengerong; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import org.apache.commons.lang.StringUtils; +import org.apache.http.*; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIUtils; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicHeader; +import org.apache.http.message.HeaderGroup; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.Closeable; +import java.io.IOException; +import java.io.PrintWriter; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import static com.google.common.collect.FluentIterable.from; +import static org.apache.commons.lang.StringUtils.isNotBlank; +import static org.apache.http.HttpHeaders.CONTENT_LENGTH; +import static org.apache.http.HttpHeaders.HOST; + +public class PrerenderSeoService { + private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); + /** + * These are the "hop-by-hop" headers that should not be copied. + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html + * I use an HttpClient HeaderGroup class instead of Set because this + * approach does case insensitive lookup faster. + */ + private static final HeaderGroup hopByHopHeaders; + public static final String ESCAPED_FRAGMENT_KEY = "_escaped_fragment_"; + private CloseableHttpClient httpClient; + private PrerenderConfig prerenderConfig; + private PreRenderEventHandler preRenderEventHandler; + + public PrerenderSeoService(Map config) { + this.prerenderConfig = new PrerenderConfig(config); + this.httpClient = getHttpClient(); + } + + static { + hopByHopHeaders = new HeaderGroup(); + String[] headers = new String[]{ + "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", + "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; + for (String header : headers) { + hopByHopHeaders.addHeader(new BasicHeader(header, null)); + } + } + + public void destroy() { + if (preRenderEventHandler != null) { + preRenderEventHandler.destroy(); + } + closeQuietly(httpClient); + } + + public boolean prerenderIfEligible(HttpServletRequest servletRequest, HttpServletResponse servletResponse) { + try { + if (handlePrerender(servletRequest, servletResponse)) { + return true; + } + } catch (Exception e) { + log.error("Prerender service error", e); + } + return false; + } + + private boolean handlePrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) + throws URISyntaxException, IOException { + if (shouldShowPrerenderedPage(servletRequest)) { + this.preRenderEventHandler = prerenderConfig.getEventHandler(); + if (beforeRender(servletRequest, servletResponse) || proxyPrerenderedPageResponse(servletRequest, servletResponse)) { + return true; + } + } + return false; + } + + private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { + final String userAgent = request.getHeader("User-Agent"); + final String url = getRequestURL(request); + final String referer = request.getHeader("Referer"); + + log.trace(String.format("checking request for %s from User-Agent %s and referer %s", url, userAgent, referer)); + + if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { + log.trace("Request is not HTTP GET; intercept: no"); + return false; + } + + if (isInResources(url)) { + log.trace("request is for a (static) resource; intercept: no"); + return false; + } + + String prerenderHeader = request.getHeader("X-Prerender"); + if (StringUtils.isNotEmpty(prerenderHeader)) { + return false; + } + + final List whiteList = prerenderConfig.getWhitelist(); + if (whiteList != null && !isInWhiteList(url, whiteList)) { + log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); + return false; + } + + final List blacklist = prerenderConfig.getBlacklist(); + if (blacklist != null && isInBlackList(url, referer, blacklist)) { + log.trace("Blacklist is enabled, and this request is listed; intercept: no"); + return false; + } + + if (hasEscapedFragment(request)) { + log.trace("Request Has _escaped_fragment_; intercept: yes"); + return true; + } + + if (StringUtils.isBlank(userAgent)) { + log.trace("Request has blank userAgent; intercept: no"); + return false; + } + + if (!isInSearchUserAgent(userAgent)) { + log.trace("Request User-Agent is not a search bot; intercept: no"); + return false; + } + + log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); + return true; + } + + protected HttpGet getHttpGet(String apiUrl) { + return new HttpGet(apiUrl); + } + + protected CloseableHttpClient getHttpClient() { + return prerenderConfig.getHttpClient(); + } + + /** + * Copy request headers from the servlet client to the proxy request. + * + * @throws java.net.URISyntaxException + */ + private void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) + throws URISyntaxException { + // Get an Enumeration of all of the header names sent by the client + Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); + while (enumerationOfHeaderNames.hasMoreElements()) { + String headerName = (String) enumerationOfHeaderNames.nextElement(); + //Instead the content-length is effectively set via InputStreamEntity + if (!headerName.equalsIgnoreCase(CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { + Enumeration headers = servletRequest.getHeaders(headerName); + while (headers.hasMoreElements()) {//sometimes more than one value + String headerValue = (String) headers.nextElement(); + // In case the proxy host is running multiple virtual servers, + // rewrite the Host header to ensure that we get content from + // the correct virtual server + if (headerName.equalsIgnoreCase(HOST)) { + HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); + headerValue = host.getHostName(); + if (host.getPort() != -1) { + headerValue += ":" + host.getPort(); + } + } + proxyRequest.addHeader(headerName, headerValue); + } + } + } + } + + private String getRequestURL(HttpServletRequest request) { + if (StringUtils.isNotEmpty(prerenderConfig.getForwardedURLHeader())) { + String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); + if (StringUtils.isNotEmpty(url)) { + return url; + } + } + if (StringUtils.isNotEmpty(prerenderConfig.getProtocol())) { + String url = request.getRequestURL().toString(); + return url.replace(request.getScheme(), prerenderConfig.getProtocol()); + } + return request.getRequestURL().toString(); + } + + private String getApiUrl(String url) { + String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); + if (!prerenderServiceUrl.endsWith("/")) { + prerenderServiceUrl += "/"; + } + return prerenderServiceUrl + url; + } + + /** + * Copy proxied response headers back to the servlet client. + */ + private void copyResponseHeaders(HttpResponse proxyResponse, final HttpServletResponse servletResponse) { + servletResponse.setCharacterEncoding(getContentCharSet(proxyResponse.getEntity())); + from(Arrays.asList(proxyResponse.getAllHeaders())).filter(new Predicate
() { + @Override + public boolean apply(Header header) { + return !hopByHopHeaders.containsHeader(header.getName()); + } + }).transform(new Function() { + @Override + public Boolean apply(Header header) { + servletResponse.addHeader(header.getName(), header.getValue()); + return true; + } + }).toList(); + } + + /** + * Get the charset used to encode the http entity. + */ + private String getContentCharSet(final HttpEntity entity) throws ParseException { + if (entity == null) { + return null; + } + String charset = null; + if (entity.getContentType() != null) { + HeaderElement values[] = entity.getContentType().getElements(); + if (values.length > 0) { + NameValuePair param = values[0].getParameterByName("charset"); + if (param != null) { + charset = param.getValue(); + } + } + } + return charset; + } + + private String getResponseHtml(HttpResponse proxyResponse) + throws IOException { + HttpEntity entity = proxyResponse.getEntity(); + return entity != null ? EntityUtils.toString(entity) : ""; + } + + /** + * Copy response body data (the entity) from the proxy to the servlet client. + */ + private void responseEntity(String html, HttpServletResponse servletResponse) + throws IOException { + PrintWriter printWriter = servletResponse.getWriter(); + try { + printWriter.write(html); + printWriter.flush(); + } finally { + closeQuietly(printWriter); + } + } + + + protected void closeQuietly(Closeable closeable) { + try { + if (closeable != null) { + closeable.close(); + } + } catch (IOException e) { + log.error("Close proxy error", e); + } + } + + private boolean hasEscapedFragment(HttpServletRequest request) { + return request.getParameterMap().containsKey(ESCAPED_FRAGMENT_KEY); + } + + private boolean isInBlackList(final String url, final String referer, List blacklist) { + return from(blacklist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + final Pattern pattern = Pattern.compile(regex); + return pattern.matcher(url).matches() || + (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); + } + }); + } + + private boolean isInSearchUserAgent(final String userAgent) { + return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return userAgent.toLowerCase().contains(item.toLowerCase()); + } + }); + } + + + private boolean isInResources(final String url) { + return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return (url.indexOf('?') >= 0 ? url.substring(0, url.indexOf('?')) : url) + .toLowerCase().endsWith(item); + } + }); + } + + private boolean isInWhiteList(final String url, List whitelist) { + return from(whitelist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + return Pattern.compile(regex).matcher(url).matches(); + } + }); + } + + private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) throws IOException { + if (preRenderEventHandler != null) { + final String html = preRenderEventHandler.beforeRender(request); + if (isNotBlank(html)) { + final PrintWriter writer = response.getWriter(); + writer.write(html); + writer.flush(); + closeQuietly(writer); + return true; + } + } + return false; + } + + private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) + throws IOException, URISyntaxException { + final String apiUrl = getApiUrl(getFullUrl(request)); + log.info(String.format("Prerender proxy will send request to:%s", apiUrl)); + final HttpGet getMethod = getHttpGet(apiUrl); + copyRequestHeaders(request, getMethod); + withPrerenderToken(getMethod); + CloseableHttpResponse prerenderServerResponse = null; + + try { + prerenderServerResponse = httpClient.execute(getMethod); + response.setStatus(prerenderServerResponse.getStatusLine().getStatusCode()); + copyResponseHeaders(prerenderServerResponse, response); + String html = getResponseHtml(prerenderServerResponse); + html = afterRender(request, response, prerenderServerResponse, html); + responseEntity(html, response); + return true; + } finally { + closeQuietly(prerenderServerResponse); + } + } + + private String afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, CloseableHttpResponse prerenderServerResponse, String responseHtml) { + if (preRenderEventHandler != null) { + return preRenderEventHandler.afterRender(clientRequest, clientResponse, prerenderServerResponse, responseHtml); + } + return responseHtml; + } + + private void withPrerenderToken(HttpRequest proxyRequest) { + final String token = prerenderConfig.getPrerenderToken(); + //for new version prerender with token. + if (isNotBlank(token)) { + proxyRequest.addHeader("X-Prerender-Token", token); + } + } + + private String getFullUrl(HttpServletRequest request) { + final String url = getRequestURL(request); + final String queryString = request.getQueryString(); + return isNotBlank(queryString) ? String.format("%s?%s", url, queryString) : url; + } +} diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 20beb9b..ba81d67 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -1,4 +1,341 @@ package com.github.greengerong; +import com.google.common.collect.Maps; +import org.apache.http.Header; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.PrintWriter; +import java.util.Enumeration; +import java.util.HashMap; + +import static com.github.greengerong.PrerenderSeoService.ESCAPED_FRAGMENT_KEY; +import static org.apache.http.HttpStatus.SC_NOT_FOUND; +import static org.apache.http.HttpStatus.SC_OK; +import static org.apache.http.client.methods.HttpGet.METHOD_NAME; +import static org.mockito.Mockito.*; + +@RunWith(MockitoJUnitRunner.class) public class PreRenderSEOFilterTest { + + private PreRenderSEOFilter preRenderSEOFilter; + + @Mock + private CloseableHttpClient httpClient; + + @Mock + private FilterConfig filterConfig; + + @Mock + private HttpServletRequest servletRequest; + + @Mock + private HttpServletResponse servletResponse; + + @Mock + private FilterChain filterChain; + + @Mock + private HttpGet httpGet; + @Mock + private PrintWriter printWriter; + + @Before + public void setUp() throws Exception { + preRenderSEOFilter = new PreRenderSEOFilter() { + @Override + public void init(FilterConfig filterConfig) throws ServletException { + setPrerenderSeoService(new PrerenderSeoService(toMap(filterConfig)) { + @Override + protected CloseableHttpClient getHttpClient() { + return httpClient; + } + + @Override + protected HttpGet getHttpGet(String apiUrl) { + return httpGet; + } + }); + } + }; + } + + @Test + public void should_not_handle_when_non_get_request() throws Exception { + //given + preRenderSEOFilter.init(filterConfig); + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer()); + when(servletRequest.getMethod()).thenReturn(HttpPost.METHOD_NAME); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_handle_when_url_with_escaped_fragment_() throws Exception { + //given + preRenderSEOFilter.init(filterConfig); + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put(ESCAPED_FRAGMENT_KEY, ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(SC_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_user_agent_is_not_crawler() throws Exception { + //given + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("no"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_x_prerender_header_is_present() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + when(servletRequest.getHeader("X-Prerender")).thenReturn("1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_url_is_a_resource() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test.js")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_white_list_is_not_empty_and_url_is_not_in_white_list() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + when(filterConfig.getInitParameter("whitelist")).thenReturn("whitelist1,whitelist2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_black_list_is_not_empty_and_url_is_in_black_list() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + when(filterConfig.getInitParameter("blacklist")).thenReturn("blacklist1,http://localhost/test"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and_white_list_is_empty_and_black_list_is_empty() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put(ESCAPED_FRAGMENT_KEY, ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(SC_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_handle_when_every_thing_is_ok_but_prerender_server_response_is_not_200() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put(ESCAPED_FRAGMENT_KEY, ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(SC_NOT_FOUND); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + verify(servletResponse).setStatus(SC_NOT_FOUND); + } + + + @Test + public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and_in_white_list_and_not_in_black_list() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + when(filterConfig.getInitParameter("whitelist")).thenReturn("whitelist1,http://localhost/test"); + when(filterConfig.getInitParameter("blacklist")).thenReturn("blacklist1,blacklist2"); + + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put(ESCAPED_FRAGMENT_KEY, ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(SC_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_use_request_url_from_custom_header_if_available() throws Exception { + //given + when(filterConfig.getInitParameter("forwardedURLHeader")).thenReturn("X-Forwarded-URL"); + when(filterConfig.getInitParameter("whitelist")).thenReturn("http://my.public.domain.com/"); + when(filterConfig.getInitParameter("blacklist")).thenReturn("http://localhost/test"); + + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getHeader("X-Forwarded-URL")).thenReturn("http://my.public.domain.com/"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put(ESCAPED_FRAGMENT_KEY, ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(SC_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } } diff --git a/src/test/java/com/github/greengerong/PrerenderConfigTest.java b/src/test/java/com/github/greengerong/PrerenderConfigTest.java new file mode 100644 index 0000000..531c021 --- /dev/null +++ b/src/test/java/com/github/greengerong/PrerenderConfigTest.java @@ -0,0 +1,35 @@ +package com.github.greengerong; + +import com.google.common.collect.Maps; +import org.apache.http.impl.client.CloseableHttpClient; +import org.junit.Test; + +import java.util.Map; + +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsNull.notNullValue; +import static org.junit.Assert.assertThat; + +public class PrerenderConfigTest { + @Test(expected = Exception.class) + public void should_throw_exception_if_invalid_timeout_value_specified() throws Exception { + //given + Map configuration = Maps.newHashMap(); + configuration.put("socketTimeout", "not_an_int"); + PrerenderConfig config = new PrerenderConfig(configuration); + //when + config.getHttpClient(); + } + + @Test + public void should_pass_if_correct_timeout_value_specified() throws Exception { + //given + Map configuration = Maps.newHashMap(); + configuration.put("socketTimeout", "1000"); + PrerenderConfig config = new PrerenderConfig(configuration); + //when + final CloseableHttpClient httpClient = config.getHttpClient(); + + assertThat(httpClient, is(notNullValue())); + } +}