diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index f8b64f1..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore index a9aaddb..112f2f2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ target/ **/target/ **/.idea/ +*.iml + +### OSX ### +.DS_Store diff --git a/.travis.yml b/.travis.yml index aec209e..e8f05af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: java script: "mvn clean install -U" jdk: - #- oraclejdk6 - - oraclejdk7 - - openjdk6 - - openjdk7 \ No newline at end of file + - openjdk7 + - openjdk8 diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..43db8ab --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017-present, GreenGerong(https://github.com/greengerong), Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 7ae6af0..4d3f00f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Prerender Java [![Build Status](https://travis-ci.org/greengerong/prerender-java.png)](https://travis-ci.org/greengerong/prerender-java) +Prerender Java =========================== Are you using backbone, angular, emberjs, etc, but you're unsure about the SEO implications? @@ -15,12 +15,8 @@ Use this java filter that prerenders a javascript-rendered page using an externa prerender com.github.greengerong.PreRenderSEOFilter - prerenderServiceUrl - http://localhost:3000 - - - crawlerUserAgents - me + prerenderToken + [get from prerender: https://prerender.io/] @@ -33,7 +29,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.6.1 + 1.6.4 ## How it works @@ -42,7 +38,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa 2. Check to make sure we aren't requesting a resource (js, css, etc...) 3. (optional) Check to make sure the url is in the whitelist 4. (optional) Check to make sure the url isn't in the blacklist -2. Make a `GET` request to the [prerender service](https://github.com/collectiveip/prerender)(phantomjs server) for the page's prerendered HTML +2. Make a `GET` request to the [prerender service](https://github.com/prerender/prerender)(phantomjs server) for the page's prerendered HTML 3. Return that HTML to the crawler ## Customization @@ -54,10 +50,16 @@ example: someproxy,someproxy1 ### blacklist +### forwardedURLHeader +Important for servers behind reverse proxy that need the public url to be used for pre-rendering. +We usually set the original url in an http header which is added by the reverse proxy (similar to the more standard `x-forwarded-proto` and `x-forwarded-for`) + +### protocol +If you specifically want to make sure that the Prerender service queries using https or http protocol, you can set the init-param `protocol` to `https` or `http` respectively. ### Using your own prerender service -If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://prerender.herokuapp.com` +If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://service.prerender.io/` $ export PRERENDER_SERVICE_URL= @@ -93,16 +95,21 @@ If you want to cache the caching, analytics, log or others, you can config it. I ## Testing -If you want to make sure your pages are rendering correctly: +If your URLs use a hash-bang: + + If you want to see `http://localhost:3000/#!/profiles/1234` + Then go to `http://localhost:3000/?_escaped_fragment_=/profiles/1234` + +If your URLs use push-state: + + If you want to see `http://localhost:3000/profiles/1234` + Then go to `http://localhost:3000/profiles/1234?_escaped_fragment_=` + -1. Open the Developer Tools in Chrome (Cmd + Atl + J) -2. Click the Settings gear in the bottom right corner. -3. Click "Overrides" on the left side of the settings panel. -4. Check the "User Agent" checkbox. -6. Choose "Other..." from the User Agent dropdown. -7. Type `googlebot` into the input box. -8. Refresh the page (make sure to keep the developer tools open). +project demo test url: + http://localhost:8080/test/?_escaped_fragment_= + ## License The MIT License (MIT) diff --git a/demo/.DS_Store b/demo/.DS_Store deleted file mode 100644 index 8b95c52..0000000 Binary files a/demo/.DS_Store and /dev/null differ diff --git a/demo/pom.xml b/demo/pom.xml index 813d096..5b1c40c 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,11 +11,32 @@ com.github.greengerong prerender-java - 1.6.1 + 1.6.4 + + + + + + + + commons-io + commons-io + 2.4 + + + org.slf4j + slf4j-log4j12 + 1.7.5 + + + log4j + log4j + 1.2.17 demo + org.mortbay.jetty diff --git a/demo/src/.DS_Store b/demo/src/.DS_Store deleted file mode 100644 index e66c6b5..0000000 Binary files a/demo/src/.DS_Store and /dev/null differ diff --git a/demo/src/main/.DS_Store b/demo/src/main/.DS_Store deleted file mode 100644 index 496035a..0000000 Binary files a/demo/src/main/.DS_Store and /dev/null differ diff --git a/demo/src/main/resources/log4j.properties b/demo/src/main/resources/log4j.properties new file mode 100644 index 0000000..3074ab0 --- /dev/null +++ b/demo/src/main/resources/log4j.properties @@ -0,0 +1,63 @@ +#log4j.rootLogger=CONSOLE,FILE +log4j.rootLogger=DEBUG,CONSOLE +log4j.addivity.org.apache=true + + +# 应用于控制台 +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=INFO +log4j.appender.CONSOLE.Target=System.out +log4j.appender.CONSOLE.Encoding=GBK +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +# 每天新建日志 +log4j.appender.A1=org.apache.log4j.DailyRollingFileAppender +log4j.appender.A1.File=C:/log4j/log +log4j.appender.A1.Encoding=GBK +log4j.appender.A1.Threshold=DEBUG +log4j.appender.A1.DatePattern='.'yyyy-MM-dd +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L : %m%n + +#应用于文件 +log4j.appender.FILE=org.apache.log4j.FileAppender +log4j.appender.FILE.File=C:/log4j/file.log +log4j.appender.FILE.Append=false +log4j.appender.FILE.Encoding=GBK +log4j.appender.FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +# 应用于文件回滚 +log4j.appender.ROLLING_FILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLING_FILE.Threshold=ERROR +log4j.appender.ROLLING_FILE.File=rolling.log +log4j.appender.ROLLING_FILE.Append=true +log4j.appender.CONSOLE_FILE.Encoding=GBK +log4j.appender.ROLLING_FILE.MaxFileSize=10KB +log4j.appender.ROLLING_FILE.MaxBackupIndex=1 +log4j.appender.ROLLING_FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLING_FILE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n + +#应用于socket +log4j.appender.SOCKET=org.apache.log4j.RollingFileAppender +log4j.appender.SOCKET.RemoteHost=localhost +log4j.appender.SOCKET.Port=5001 +log4j.appender.SOCKET.LocationInfo=true +# Set up for Log Facter 5 +log4j.appender.SOCKET.layout=org.apache.log4j.PatternLayout +log4j.appender.SOCET.layout.ConversionPattern=[start]%d{DATE}[DATE]%n%p[PRIORITY]%n%x[NDC]%n%t[THREAD]%n%c[CATEGORY]%n%m[MESSAGE]%n%n +# Log Factor 5 Appender +log4j.appender.LF5_APPENDER=org.apache.log4j.lf5.LF5Appender +log4j.appender.LF5_APPENDER.MaxNumberOfRecords=2000 + +# 发送日志给邮件 +log4j.appender.MAIL=org.apache.log4j.net.SMTPAppender +log4j.appender.MAIL.Threshold=FATAL +log4j.appender.MAIL.BufferSize=10 +log4j.appender.MAIL.From= +log4j.appender.MAIL.SMTPHost=www.wusetu.com +log4j.appender.MAIL.Subject=Log4J Message +log4j.appender.MAIL.To= +log4j.appender.MAIL.layout=org.apache.log4j.PatternLayout +log4j.appender.MAIL.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n diff --git a/demo/src/main/webapp/.DS_Store b/demo/src/main/webapp/.DS_Store deleted file mode 100644 index 2a74ced..0000000 Binary files a/demo/src/main/webapp/.DS_Store and /dev/null differ diff --git a/demo/src/main/webapp/WEB-INF/web.xml b/demo/src/main/webapp/WEB-INF/web.xml index eadd7d5..2677fca 100644 --- a/demo/src/main/webapp/WEB-INF/web.xml +++ b/demo/src/main/webapp/WEB-INF/web.xml @@ -15,6 +15,10 @@ crawlerUserAgents YahooSeeker + + + + prerender diff --git a/pom.xml b/pom.xml index 47e46f1..01017d3 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.2-SNAPSHOT + 1.6.5-SNAPSHOT jar prerender-java @@ -15,6 +15,7 @@ http://www.apache.org/licenses/LICENSE-2.0.txt repo + scm:git:git@github.com:greengerong/prerender-java.git @@ -90,12 +91,12 @@ sonatype-nexus-snapshots Sonatype Nexus Snapshots - http://oss.sonatype.org/content/repositories/snapshots + https://oss.sonatype.org/content/repositories/snapshots sonatype-nexus-staging Nexus Release Repository - http://oss.sonatype.org/service/local/staging/deploy/maven2/ + https://oss.sonatype.org/service/local/staging/deploy/maven2/ @@ -114,9 +115,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.7 - - + 2.8.1 attach-javadoc diff --git a/prerender-java.iml b/prerender-java.iml index cbd3c69..f74aa01 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -23,5 +23,4 @@ - - + \ No newline at end of file diff --git a/src/main/.DS_Store b/src/main/.DS_Store deleted file mode 100644 index 1310f12..0000000 Binary files a/src/main/.DS_Store and /dev/null differ diff --git a/src/main/java/.DS_Store b/src/main/java/.DS_Store deleted file mode 100644 index 666f71f..0000000 Binary files a/src/main/java/.DS_Store and /dev/null differ diff --git a/src/main/java/com/.DS_Store b/src/main/java/com/.DS_Store deleted file mode 100644 index e0512af..0000000 Binary files a/src/main/java/com/.DS_Store and /dev/null differ diff --git a/src/main/java/com/github/.DS_Store b/src/main/java/com/github/.DS_Store deleted file mode 100644 index 162ded6..0000000 Binary files a/src/main/java/com/github/.DS_Store and /dev/null differ diff --git a/src/main/java/com/github/greengerong/PreRenderEventHandler.java b/src/main/java/com/github/greengerong/PreRenderEventHandler.java index 9e0d74e..bd71634 100644 --- a/src/main/java/com/github/greengerong/PreRenderEventHandler.java +++ b/src/main/java/com/github/greengerong/PreRenderEventHandler.java @@ -4,10 +4,13 @@ import org.apache.http.HttpResponse; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; public interface PreRenderEventHandler { String beforeRender(HttpServletRequest clientRequest); - void afterRender(HttpServletRequest clientRequest, HttpResponse prerenderResponse); + String afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, HttpResponse prerenderResponse, String responseHtml); + + void destroy(); } diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 4350fa6..5704025 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,321 +1,55 @@ package com.github.greengerong; - -import com.google.common.base.Predicate; -import org.apache.commons.lang.StringUtils; -import org.apache.http.*; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIUtils; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.message.BasicHeader; -import org.apache.http.message.HeaderGroup; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.servlet.*; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.Closeable; import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintWriter; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Enumeration; import java.util.List; -import java.util.regex.Pattern; - -import static com.google.common.collect.FluentIterable.from; +import java.util.Map; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; public class PreRenderSEOFilter implements Filter { - - private final static Logger log = LoggerFactory.getLogger(PreRenderSEOFilter.class); - private CloseableHttpClient httpClient; - private PreRenderEventHandler preRenderEventHandler; - private PrerenderConfig prerenderConfig; - public static final int HTTP_OK = 200; + public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", + "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", + "blacklist", "prerenderServiceUrl", "protocol","socketTimeout"); + private PrerenderSeoService prerenderSeoService; @Override public void init(FilterConfig filterConfig) throws ServletException { - this.prerenderConfig = new PrerenderConfig(filterConfig); - this.httpClient = getHttpClient(); - this.preRenderEventHandler = prerenderConfig.getEventHandler(); - } - - protected CloseableHttpClient getHttpClient() { - return prerenderConfig.getHttpClient(); + this.prerenderSeoService = new PrerenderSeoService(toMap(filterConfig)); } @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - try { - final HttpServletRequest request = (HttpServletRequest) servletRequest; - final HttpServletResponse response = (HttpServletResponse) servletResponse; - if (shouldShowPrerenderedPage(request)) { - - if (beforeRender(request, response) || proxyPrerenderedPageResponse(request, response)) { - return; - } - } - } catch (Exception e) { - log.error("Prerender service error", e); + boolean isPreRendered = prerenderSeoService.prerenderIfEligible( + (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); + if (!isPreRendered) { + filterChain.doFilter(servletRequest, servletResponse); } - filterChain.doFilter(servletRequest, servletResponse); - } - - private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) throws IOException { - if (preRenderEventHandler != null) { - final String html = preRenderEventHandler.beforeRender(request); - if (StringUtils.isNotBlank(html)) { - final PrintWriter writer = response.getWriter(); - writer.write(html); - writer.flush(); - closeQuietly(writer); - return true; - } - } - return false; - } - - private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException { - final String apiUrl = getApiUrl(getFullUrl(request)); - log.trace(String.format("Prerender proxy will send request to:%s", apiUrl)); - final HttpGet getMethod = getHttpGet(apiUrl); - copyRequestHeaders(request, getMethod); - withPrerenderToken(getMethod); - CloseableHttpResponse proxyResponse = null; - - try { - proxyResponse = httpClient.execute(getMethod); - if (proxyResponse.getStatusLine().getStatusCode() == HTTP_OK) { - afterRender(request, proxyResponse); - copyResponseHeaders(proxyResponse, response); - copyResponseEntity(proxyResponse, response); - return true; - } - } finally { - closeQuietly(proxyResponse); - } - return false; - } - - protected HttpGet getHttpGet(String apiUrl) { - return new HttpGet(apiUrl); - } - - private void afterRender(HttpServletRequest request, CloseableHttpResponse proxyResponse) { - if (preRenderEventHandler != null) { - preRenderEventHandler.afterRender(request, proxyResponse); - } - } - - private void withPrerenderToken(HttpRequest proxyRequest) { - final String token = prerenderConfig.getPrerenderToken(); - //for new version prerender with token. - if (StringUtils.isNotBlank(token)) { - proxyRequest.addHeader("X-Prerender-Token", token); - } - } - - /** - * Copy proxied response headers back to the servlet client. - */ - protected void copyResponseHeaders(HttpResponse proxyResponse, HttpServletResponse servletResponse) { - for (Header header : proxyResponse.getAllHeaders()) { - if (!hopByHopHeaders.containsHeader(header.getName())) { - servletResponse.addHeader(header.getName(), header.getValue()); - } - } - } - - /** - * Copy response body data (the entity) from the proxy to the servlet client. - */ - protected void copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse) throws IOException { - HttpEntity entity = proxyResponse.getEntity(); - if (entity != null) { - OutputStream servletOutputStream = servletResponse.getOutputStream(); - try { - entity.writeTo(servletOutputStream); - } finally { - closeQuietly(servletOutputStream); - } - } - } - - protected void closeQuietly(Closeable closeable) { - try { - if (closeable != null) { - closeable.close(); - } - } catch (IOException e) { - log.error("Close proxy error", e); - } - } - - /** - * These are the "hop-by-hop" headers that should not be copied. - * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html - * I use an HttpClient HeaderGroup class instead of Set because this - * approach does case insensitive lookup faster. - */ - protected static final HeaderGroup hopByHopHeaders; - - static { - hopByHopHeaders = new HeaderGroup(); - String[] headers = new String[]{ - "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", - "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; - for (String header : headers) { - hopByHopHeaders.addHeader(new BasicHeader(header, null)); - } - } - - /** - * Copy request headers from the servlet client to the proxy request. - * - * @throws URISyntaxException - */ - protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) throws URISyntaxException { - // Get an Enumeration of all of the header names sent by the client - Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); - while (enumerationOfHeaderNames.hasMoreElements()) { - String headerName = (String) enumerationOfHeaderNames.nextElement(); - //Instead the content-length is effectively set via InputStreamEntity - if (!headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { - Enumeration headers = servletRequest.getHeaders(headerName); - while (headers.hasMoreElements()) {//sometimes more than one value - String headerValue = (String) headers.nextElement(); - // In case the proxy host is running multiple virtual servers, - // rewrite the Host header to ensure that we get content from - // the correct virtual server - if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) { - HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); - headerValue = host.getHostName(); - if (host.getPort() != -1) { - headerValue += ":" + host.getPort(); - } - } - proxyRequest.addHeader(headerName, headerValue); - } - } - } - } - - private String getFullUrl(HttpServletRequest request) { - final StringBuffer url = request.getRequestURL(); - final String queryString = request.getQueryString(); - if (queryString != null) { - url.append('?'); - url.append(queryString); - } - return url.toString(); } @Override public void destroy() { - prerenderConfig = null; - closeQuietly(httpClient); - } - - private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { - final String userAgent = request.getHeader("User-Agent"); - final String url = request.getRequestURL().toString(); - final String referer = request.getHeader("Referer"); - - log.trace("checking request for " + url + " from User-Agent " + userAgent + " and referer " + referer); - - if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { - log.trace("Request is not HTTP GET; intercept: no"); - return false; - } - - if (isInResources(url)) { - log.trace("request is for a (static) resource; intercept: no"); - return false; - } - - final List whiteList = prerenderConfig.getWhitelist(); - if (whiteList != null && !isInWhiteList(url, whiteList)) { - log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); - return false; - } - - final List blacklist = prerenderConfig.getBlacklist(); - if (blacklist != null && isInBlackList(url, referer, blacklist)) { - log.trace("Blacklist is enabled, and this request is listed; intercept: no"); - return false; - } - - if (hasEscapedFragment(request)) { - log.trace("Request Has _escaped_fragment_; intercept: yes"); - return true; - } - - if (StringUtils.isBlank(userAgent)) { - log.trace("Request has blank userAgent; intercept: no"); - return false; - } - - if (!isInSearchUserAgent(userAgent)) { - log.trace("Request User-Agent is not a search bot; intercept: no"); - return false; - } - - log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); - return true; + prerenderSeoService.destroy(); } - private boolean hasEscapedFragment(HttpServletRequest request) { - return request.getParameterMap().containsKey("_escaped_fragment_"); + protected void setPrerenderSeoService(PrerenderSeoService prerenderSeoService) { + this.prerenderSeoService = prerenderSeoService; } - private String getApiUrl(String url) { - String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); - if (!prerenderServiceUrl.endsWith("/")) { - prerenderServiceUrl += "/"; + protected Map toMap(FilterConfig filterConfig) { + Map config = Maps.newHashMap(); + for (String parameterName : PARAMETER_NAMES) { + config.put(parameterName, filterConfig.getInitParameter(parameterName)); } - return prerenderServiceUrl + url; - } - - private boolean isInBlackList(final String url, final String referer, List blacklist) { - return from(blacklist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - final Pattern pattern = Pattern.compile(regex); - return pattern.matcher(url).matches() || - (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); - } - }); - } - - private boolean isInWhiteList(final String url, List whitelist) { - return from(whitelist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - return Pattern.compile(regex).matcher(url).matches(); - } - }); - } - - private boolean isInResources(final String url) { - return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return url.contains(item.toLowerCase()); - } - }); + return config; } - - private boolean isInSearchUserAgent(final String userAgent) { - return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return userAgent.toLowerCase().indexOf(item.toLowerCase()) >= 0; - } - }); - } - } + diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index c5daa05..6b4e894 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -2,8 +2,8 @@ import com.google.common.collect.Lists; -import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; +import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; @@ -12,21 +12,24 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.FilterConfig; import java.util.Arrays; import java.util.List; +import java.util.Map; + +import static org.apache.commons.lang.StringUtils.isNotBlank; public class PrerenderConfig { private final static Logger log = LoggerFactory.getLogger(PrerenderConfig.class); - private FilterConfig filterConfig; + public static final String PRERENDER_IO_SERVICE_URL = "http://service.prerender.io/"; + private Map config; - public PrerenderConfig(FilterConfig filterConfig) { - this.filterConfig = filterConfig; + public PrerenderConfig(Map config) { + this.config = config; } public PreRenderEventHandler getEventHandler() { - final String preRenderEventHandler = filterConfig.getInitParameter("preRenderEventHandler"); - if (StringUtils.isNotBlank(preRenderEventHandler)) { + final String preRenderEventHandler = config.get("preRenderEventHandler"); + if (isNotBlank(preRenderEventHandler)) { try { return (PreRenderEventHandler) Class.forName(preRenderEventHandler).newInstance(); } catch (Exception e) { @@ -37,27 +40,59 @@ public PreRenderEventHandler getEventHandler() { } public CloseableHttpClient getHttpClient() { - HttpClientBuilder builder = HttpClients.custom(); - final String proxy = filterConfig.getInitParameter("proxy"); - if (StringUtils.isNotBlank(proxy)) { - final int proxyPort = Integer.parseInt(filterConfig.getInitParameter("proxyPort")); + HttpClientBuilder builder = HttpClients.custom() + .setConnectionManager(new PoolingHttpClientConnectionManager()) + .disableRedirectHandling(); + + configureProxy(builder); + configureTimeout(builder); + return builder.build(); + } + + private HttpClientBuilder configureProxy(HttpClientBuilder builder) { + final String proxy = config.get("proxy"); + if (isNotBlank(proxy)) { + final int proxyPort = Integer.parseInt(config.get("proxyPort")); DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); - builder = builder.setRoutePlanner(routePlanner); + builder.setRoutePlanner(routePlanner); } + return builder; + } - builder = builder.setConnectionManager(new PoolingHttpClientConnectionManager()); - return builder.build(); + private HttpClientBuilder configureTimeout(HttpClientBuilder builder) { + final String socketTimeout = getSocketTimeout(); + if (socketTimeout != null) { + RequestConfig config = RequestConfig.custom().setSocketTimeout(Integer.parseInt(socketTimeout)).build(); + builder.setDefaultRequestConfig(config); + } + return builder; + } + + public String getSocketTimeout() { + return config.get("socketTimeout"); } public String getPrerenderToken() { - return filterConfig.getInitParameter("prerenderToken"); + return config.get("prerenderToken"); + } + + public String getForwardedURLHeader() { + return config.get("forwardedURLHeader"); + } + + public String getProtocol() { + return config.get("protocol"); } public List getCrawlerUserAgents() { - List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", - "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly"); - final String crawlerUserAgentsFromConfig = filterConfig.getInitParameter("crawlerUserAgents"); - if (StringUtils.isNotBlank(crawlerUserAgentsFromConfig)) { + List crawlerUserAgents = Lists.newArrayList("baiduspider", + "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview" + , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot", "vkShare", + "W3C_Validator", "redditbot", "Applebot", "whatsapp", "flipboard", "tumblr", "bitlybot", + "skypeuripreview", "nuzzel", "discordbot", "google page speed", "qwantify", "pinterestbot", + "bitrix link preview", "xing-contenttabreceiver", "chrome-lighthouse", "telegrambot"); + final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); + if (isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); } @@ -65,12 +100,12 @@ public List getCrawlerUserAgents() { } public List getExtensionsToIgnore() { - List extensionsToIgnore = Lists.newArrayList(".js", ".css", ".less", ".png", ".jpg", ".jpeg", - ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", - ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", - ".iso", ".flv", ".m4v", ".torrent"); - final String extensionsToIgnoreFromConfig = filterConfig.getInitParameter("extensionsToIgnore"); - if (StringUtils.isNotBlank(extensionsToIgnoreFromConfig)) { + List extensionsToIgnore = Lists.newArrayList(".js", ".json", ".css", ".xml", ".less", ".png", ".jpg", + ".jpeg", ".gif", ".pdf", ".doc", ".txt", ".ico", ".rss", ".zip", ".mp3", ".rar", ".exe", ".wmv", + ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", + ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent", ".woff", ".ttf"); + final String extensionsToIgnoreFromConfig = config.get("extensionsToIgnore"); + if (isNotBlank(extensionsToIgnoreFromConfig)) { extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); } @@ -78,23 +113,28 @@ public List getExtensionsToIgnore() { } public List getWhitelist() { - final String whitelist = filterConfig.getInitParameter("whitelist"); - if (StringUtils.isNotBlank(whitelist)) { + final String whitelist = config.get("whitelist"); + if (isNotBlank(whitelist)) { return Arrays.asList(whitelist.trim().split(",")); } return null; } public List getBlacklist() { - final String blacklist = filterConfig.getInitParameter("blacklist"); - if (StringUtils.isNotBlank(blacklist)) { + final String blacklist = config.get("blacklist"); + if (isNotBlank(blacklist)) { return Arrays.asList(blacklist.trim().split(",")); } return null; } public String getPrerenderServiceUrl() { - final String prerenderServiceUrl = filterConfig.getInitParameter("prerenderServiceUrl"); - return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : "http://service.prerender.io/"; + final String prerenderServiceUrl = config.get("prerenderServiceUrl"); + return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : getDefaultPrerenderIoServiceUrl(); + } + + private String getDefaultPrerenderIoServiceUrl() { + final String prerenderServiceUrlInEnv = System.getProperty("PRERENDER_SERVICE_URL"); + return isNotBlank(prerenderServiceUrlInEnv) ? prerenderServiceUrlInEnv : PRERENDER_IO_SERVICE_URL; } } diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java new file mode 100644 index 0000000..63d3352 --- /dev/null +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -0,0 +1,378 @@ +package com.github.greengerong; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import org.apache.commons.lang.StringUtils; +import org.apache.http.*; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIUtils; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicHeader; +import org.apache.http.message.HeaderGroup; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.Closeable; +import java.io.IOException; +import java.io.PrintWriter; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import static com.google.common.collect.FluentIterable.from; +import static org.apache.commons.lang.StringUtils.isNotBlank; +import static org.apache.http.HttpHeaders.CONTENT_LENGTH; +import static org.apache.http.HttpHeaders.HOST; + +public class PrerenderSeoService { + private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); + /** + * These are the "hop-by-hop" headers that should not be copied. + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html + * I use an HttpClient HeaderGroup class instead of Set because this + * approach does case insensitive lookup faster. + */ + private static final HeaderGroup hopByHopHeaders; + public static final String ESCAPED_FRAGMENT_KEY = "_escaped_fragment_"; + private CloseableHttpClient httpClient; + private PrerenderConfig prerenderConfig; + private PreRenderEventHandler preRenderEventHandler; + + public PrerenderSeoService(Map config) { + this.prerenderConfig = new PrerenderConfig(config); + this.httpClient = getHttpClient(); + } + + static { + hopByHopHeaders = new HeaderGroup(); + String[] headers = new String[]{ + "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", + "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; + for (String header : headers) { + hopByHopHeaders.addHeader(new BasicHeader(header, null)); + } + } + + public void destroy() { + if (preRenderEventHandler != null) { + preRenderEventHandler.destroy(); + } + closeQuietly(httpClient); + } + + public boolean prerenderIfEligible(HttpServletRequest servletRequest, HttpServletResponse servletResponse) { + try { + if (handlePrerender(servletRequest, servletResponse)) { + return true; + } + } catch (Exception e) { + log.error("Prerender service error", e); + } + return false; + } + + private boolean handlePrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) + throws URISyntaxException, IOException { + if (shouldShowPrerenderedPage(servletRequest)) { + this.preRenderEventHandler = prerenderConfig.getEventHandler(); + if (beforeRender(servletRequest, servletResponse) || proxyPrerenderedPageResponse(servletRequest, servletResponse)) { + return true; + } + } + return false; + } + + private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { + final String userAgent = request.getHeader("User-Agent"); + final String url = getRequestURL(request); + final String referer = request.getHeader("Referer"); + + log.trace(String.format("checking request for %s from User-Agent %s and referer %s", url, userAgent, referer)); + + if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { + log.trace("Request is not HTTP GET; intercept: no"); + return false; + } + + if (isInResources(url)) { + log.trace("request is for a (static) resource; intercept: no"); + return false; + } + + String prerenderHeader = request.getHeader("X-Prerender"); + if (StringUtils.isNotEmpty(prerenderHeader)) { + return false; + } + + final List whiteList = prerenderConfig.getWhitelist(); + if (whiteList != null && !isInWhiteList(url, whiteList)) { + log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); + return false; + } + + final List blacklist = prerenderConfig.getBlacklist(); + if (blacklist != null && isInBlackList(url, referer, blacklist)) { + log.trace("Blacklist is enabled, and this request is listed; intercept: no"); + return false; + } + + if (hasEscapedFragment(request)) { + log.trace("Request Has _escaped_fragment_; intercept: yes"); + return true; + } + + if (StringUtils.isBlank(userAgent)) { + log.trace("Request has blank userAgent; intercept: no"); + return false; + } + + if (!isInSearchUserAgent(userAgent)) { + log.trace("Request User-Agent is not a search bot; intercept: no"); + return false; + } + + log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); + return true; + } + + protected HttpGet getHttpGet(String apiUrl) { + return new HttpGet(apiUrl); + } + + protected CloseableHttpClient getHttpClient() { + return prerenderConfig.getHttpClient(); + } + + /** + * Copy request headers from the servlet client to the proxy request. + * + * @throws java.net.URISyntaxException + */ + private void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) + throws URISyntaxException { + // Get an Enumeration of all of the header names sent by the client + Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); + while (enumerationOfHeaderNames.hasMoreElements()) { + String headerName = (String) enumerationOfHeaderNames.nextElement(); + //Instead the content-length is effectively set via InputStreamEntity + if (!headerName.equalsIgnoreCase(CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { + Enumeration headers = servletRequest.getHeaders(headerName); + while (headers.hasMoreElements()) {//sometimes more than one value + String headerValue = (String) headers.nextElement(); + // In case the proxy host is running multiple virtual servers, + // rewrite the Host header to ensure that we get content from + // the correct virtual server + if (headerName.equalsIgnoreCase(HOST)) { + HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); + headerValue = host.getHostName(); + if (host.getPort() != -1) { + headerValue += ":" + host.getPort(); + } + } + proxyRequest.addHeader(headerName, headerValue); + } + } + } + } + + private String getRequestURL(HttpServletRequest request) { + if (StringUtils.isNotEmpty(prerenderConfig.getForwardedURLHeader())) { + String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); + if (StringUtils.isNotEmpty(url)) { + return url; + } + } + if (StringUtils.isNotEmpty(prerenderConfig.getProtocol())) { + String url = request.getRequestURL().toString(); + return url.replace(request.getScheme(), prerenderConfig.getProtocol()); + } + return request.getRequestURL().toString(); + } + + private String getApiUrl(String url) { + String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); + if (!prerenderServiceUrl.endsWith("/")) { + prerenderServiceUrl += "/"; + } + return prerenderServiceUrl + url; + } + + /** + * Copy proxied response headers back to the servlet client. + */ + private void copyResponseHeaders(HttpResponse proxyResponse, final HttpServletResponse servletResponse) { + servletResponse.setCharacterEncoding(getContentCharSet(proxyResponse.getEntity())); + from(Arrays.asList(proxyResponse.getAllHeaders())).filter(new Predicate
() { + @Override + public boolean apply(Header header) { + return !hopByHopHeaders.containsHeader(header.getName()); + } + }).transform(new Function() { + @Override + public Boolean apply(Header header) { + servletResponse.addHeader(header.getName(), header.getValue()); + return true; + } + }).toList(); + } + + /** + * Get the charset used to encode the http entity. + */ + private String getContentCharSet(final HttpEntity entity) throws ParseException { + if (entity == null) { + return null; + } + String charset = null; + if (entity.getContentType() != null) { + HeaderElement values[] = entity.getContentType().getElements(); + if (values.length > 0) { + NameValuePair param = values[0].getParameterByName("charset"); + if (param != null) { + charset = param.getValue(); + } + } + } + return charset; + } + + private String getResponseHtml(HttpResponse proxyResponse) + throws IOException { + HttpEntity entity = proxyResponse.getEntity(); + return entity != null ? EntityUtils.toString(entity) : ""; + } + + /** + * Copy response body data (the entity) from the proxy to the servlet client. + */ + private void responseEntity(String html, HttpServletResponse servletResponse) + throws IOException { + PrintWriter printWriter = servletResponse.getWriter(); + try { + printWriter.write(html); + printWriter.flush(); + } finally { + closeQuietly(printWriter); + } + } + + + protected void closeQuietly(Closeable closeable) { + try { + if (closeable != null) { + closeable.close(); + } + } catch (IOException e) { + log.error("Close proxy error", e); + } + } + + private boolean hasEscapedFragment(HttpServletRequest request) { + return request.getParameterMap().containsKey(ESCAPED_FRAGMENT_KEY); + } + + private boolean isInBlackList(final String url, final String referer, List blacklist) { + return from(blacklist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + final Pattern pattern = Pattern.compile(regex); + return pattern.matcher(url).matches() || + (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); + } + }); + } + + private boolean isInSearchUserAgent(final String userAgent) { + return from(prerenderConfig.getCrawlerUserAgents()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return userAgent.toLowerCase().contains(item.toLowerCase()); + } + }); + } + + + private boolean isInResources(final String url) { + return from(prerenderConfig.getExtensionsToIgnore()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return (url.indexOf('?') >= 0 ? url.substring(0, url.indexOf('?')) : url) + .toLowerCase().endsWith(item); + } + }); + } + + private boolean isInWhiteList(final String url, List whitelist) { + return from(whitelist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + return Pattern.compile(regex).matcher(url).matches(); + } + }); + } + + private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) throws IOException { + if (preRenderEventHandler != null) { + final String html = preRenderEventHandler.beforeRender(request); + if (isNotBlank(html)) { + final PrintWriter writer = response.getWriter(); + writer.write(html); + writer.flush(); + closeQuietly(writer); + return true; + } + } + return false; + } + + private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) + throws IOException, URISyntaxException { + final String apiUrl = getApiUrl(getFullUrl(request)); + log.info(String.format("Prerender proxy will send request to:%s", apiUrl)); + final HttpGet getMethod = getHttpGet(apiUrl); + copyRequestHeaders(request, getMethod); + withPrerenderToken(getMethod); + CloseableHttpResponse prerenderServerResponse = null; + + try { + prerenderServerResponse = httpClient.execute(getMethod); + response.setStatus(prerenderServerResponse.getStatusLine().getStatusCode()); + copyResponseHeaders(prerenderServerResponse, response); + String html = getResponseHtml(prerenderServerResponse); + html = afterRender(request, response, prerenderServerResponse, html); + responseEntity(html, response); + return true; + } finally { + closeQuietly(prerenderServerResponse); + } + } + + private String afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, CloseableHttpResponse prerenderServerResponse, String responseHtml) { + if (preRenderEventHandler != null) { + return preRenderEventHandler.afterRender(clientRequest, clientResponse, prerenderServerResponse, responseHtml); + } + return responseHtml; + } + + private void withPrerenderToken(HttpRequest proxyRequest) { + final String token = prerenderConfig.getPrerenderToken(); + //for new version prerender with token. + if (isNotBlank(token)) { + proxyRequest.addHeader("X-Prerender-Token", token); + } + } + + private String getFullUrl(HttpServletRequest request) { + final String url = getRequestURL(request); + final String queryString = request.getQueryString(); + return isNotBlank(queryString) ? String.format("%s?%s", url, queryString) : url; + } +} diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 1ac7521..ba81d67 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -15,17 +15,22 @@ import javax.servlet.FilterChain; import javax.servlet.FilterConfig; +import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.PrintWriter; import java.util.Enumeration; import java.util.HashMap; +import static com.github.greengerong.PrerenderSeoService.ESCAPED_FRAGMENT_KEY; +import static org.apache.http.HttpStatus.SC_NOT_FOUND; +import static org.apache.http.HttpStatus.SC_OK; +import static org.apache.http.client.methods.HttpGet.METHOD_NAME; import static org.mockito.Mockito.*; @RunWith(MockitoJUnitRunner.class) public class PreRenderSEOFilterTest { - public static final int NOT_FOUND = 404; private PreRenderSEOFilter preRenderSEOFilter; @Mock @@ -45,18 +50,25 @@ public class PreRenderSEOFilterTest { @Mock private HttpGet httpGet; + @Mock + private PrintWriter printWriter; @Before public void setUp() throws Exception { preRenderSEOFilter = new PreRenderSEOFilter() { @Override - protected CloseableHttpClient getHttpClient() { - return httpClient; - } - - @Override - protected HttpGet getHttpGet(String apiUrl) { - return httpGet; + public void init(FilterConfig filterConfig) throws ServletException { + setPrerenderSeoService(new PrerenderSeoService(toMap(filterConfig)) { + @Override + protected CloseableHttpClient getHttpClient() { + return httpClient; + } + + @Override + protected HttpGet getHttpGet(String apiUrl) { + return httpGet; + } + }); } }; } @@ -84,15 +96,17 @@ public void should_handle_when_url_with_escaped_fragment_() throws Exception { final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(SC_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -107,7 +121,7 @@ public void should_not_handle_when_user_agent_is_not_crawler() throws Exception preRenderSEOFilter.init(filterConfig); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); when(servletRequest.getHeader("User-Agent")).thenReturn("no"); //when @@ -118,6 +132,25 @@ public void should_not_handle_when_user_agent_is_not_crawler() throws Exception verify(filterChain).doFilter(servletRequest, servletResponse); } + @Test + public void should_not_handle_when_x_prerender_header_is_present() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + when(servletRequest.getHeader("X-Prerender")).thenReturn("1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + @Test public void should_not_handle_when_url_is_a_resource() throws Exception { //given @@ -125,7 +158,7 @@ public void should_not_handle_when_url_is_a_resource() throws Exception { preRenderSEOFilter.init(filterConfig); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test.js")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); //when @@ -144,7 +177,7 @@ public void should_not_handle_when_white_list_is_not_empty_and_url_is_not_in_whi preRenderSEOFilter.init(filterConfig); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); //when @@ -163,7 +196,7 @@ public void should_not_handle_when_black_list_is_not_empty_and_url_is_in_black_l preRenderSEOFilter.init(filterConfig); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); //when @@ -184,18 +217,18 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(SC_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); - + when(servletResponse.getWriter()).thenReturn(printWriter); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); @@ -205,7 +238,7 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and } @Test - public void should_not_handle_when_every_thing_is_ok_but_prerender_server_response_is_not_200() throws Exception { + public void should_handle_when_every_thing_is_ok_but_prerender_server_response_is_not_200() throws Exception { //given when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); preRenderSEOFilter.init(filterConfig); @@ -214,23 +247,27 @@ public void should_not_handle_when_every_thing_is_ok_but_prerender_server_respon final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(NOT_FOUND); + when(statusLine.getStatusCode()).thenReturn(SC_NOT_FOUND); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); //then verify(httpClient).execute(httpGet); - verify(filterChain).doFilter(servletRequest, servletResponse); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + verify(servletResponse).setStatus(SC_NOT_FOUND); } @@ -247,17 +284,52 @@ public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and final StatusLine statusLine = mock(StatusLine.class); when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(HttpGet.METHOD_NAME); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); when(httpClient.execute(httpGet)).thenReturn(httpResponse); when(httpResponse.getStatusLine()).thenReturn(statusLine); final HashMap map = Maps.newHashMap(); - map.put("_escaped_fragment_", ""); + map.put(ESCAPED_FRAGMENT_KEY, ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(SC_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_use_request_url_from_custom_header_if_available() throws Exception { + //given + when(filterConfig.getInitParameter("forwardedURLHeader")).thenReturn("X-Forwarded-URL"); + when(filterConfig.getInitParameter("whitelist")).thenReturn("http://my.public.domain.com/"); + when(filterConfig.getInitParameter("blacklist")).thenReturn("http://localhost/test"); + + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getHeader("X-Forwarded-URL")).thenReturn("http://my.public.domain.com/"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put(ESCAPED_FRAGMENT_KEY, ""); when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(PreRenderSEOFilter.HTTP_OK); + when(statusLine.getStatusCode()).thenReturn(SC_OK); when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); //when preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); diff --git a/src/test/java/com/github/greengerong/PrerenderConfigTest.java b/src/test/java/com/github/greengerong/PrerenderConfigTest.java new file mode 100644 index 0000000..531c021 --- /dev/null +++ b/src/test/java/com/github/greengerong/PrerenderConfigTest.java @@ -0,0 +1,35 @@ +package com.github.greengerong; + +import com.google.common.collect.Maps; +import org.apache.http.impl.client.CloseableHttpClient; +import org.junit.Test; + +import java.util.Map; + +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsNull.notNullValue; +import static org.junit.Assert.assertThat; + +public class PrerenderConfigTest { + @Test(expected = Exception.class) + public void should_throw_exception_if_invalid_timeout_value_specified() throws Exception { + //given + Map configuration = Maps.newHashMap(); + configuration.put("socketTimeout", "not_an_int"); + PrerenderConfig config = new PrerenderConfig(configuration); + //when + config.getHttpClient(); + } + + @Test + public void should_pass_if_correct_timeout_value_specified() throws Exception { + //given + Map configuration = Maps.newHashMap(); + configuration.put("socketTimeout", "1000"); + PrerenderConfig config = new PrerenderConfig(configuration); + //when + final CloseableHttpClient httpClient = config.getHttpClient(); + + assertThat(httpClient, is(notNullValue())); + } +}