diff --git a/pom.xml b/pom.xml index 98419ae..06085c1 100644 --- a/pom.xml +++ b/pom.xml @@ -3,13 +3,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - com.github.philipjkim + com.github.greengerong prerender-java - 1.0.1 + 2.1.1 jar prerender-java - https://github.com/philipjkim/prerender-java + https://github.com/CatchTable/prerender-java @@ -20,9 +20,9 @@ - scm:git:git@github.com:philipjkim/prerender-java.git - scm:git:git@github.com:philipjkim/prerender-java.git - git@github.com:philipjkim/prerender-java.git + scm:git:git@github.com:CatchTable/prerender-java.git + scm:git:git@github.com:CatchTable/prerender-java.git + git@github.com:CatchTable/prerender-java.git @@ -104,8 +104,8 @@ prerender-java - GitHub philipjkim Apache Maven Packages - https://maven.pkg.github.com/philipjkim/prerender-java + GitHub Packages + https://maven.pkg.github.com/CatchTable/prerender-java diff --git a/src/main/java/com/github/greengerong/PreRenderEventHandler.java b/src/main/java/com/github/greengerong/PreRenderEventHandler.java index bd71634..ac3c52d 100644 --- a/src/main/java/com/github/greengerong/PreRenderEventHandler.java +++ b/src/main/java/com/github/greengerong/PreRenderEventHandler.java @@ -1,16 +1,16 @@ package com.github.greengerong; -import org.apache.http.HttpResponse; - import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.http.HttpResponse; public interface PreRenderEventHandler { - String beforeRender(HttpServletRequest clientRequest); + String beforeRender(HttpServletRequest clientRequest); - String afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, HttpResponse prerenderResponse, String responseHtml); + String afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, + HttpResponse prerenderResponse, String responseHtml); - void destroy(); + void destroy(); } diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 39195f9..d85cc1d 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,5 +1,7 @@ package com.github.greengerong; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import java.io.IOException; import java.util.List; import java.util.Map; @@ -11,45 +13,47 @@ import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; public class PreRenderSEOFilter implements Filter { - public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", - "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl", "protocol"); - private PrerenderSeoService prerenderSeoService; - - @Override - public void init(FilterConfig filterConfig) throws ServletException { - this.prerenderSeoService = new PrerenderSeoService(toMap(filterConfig)); - } - @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) - throws IOException, ServletException { - boolean isPrerendered = prerenderSeoService.prerenderIfEligible( - (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); - if (!isPrerendered) { - filterChain.doFilter(servletRequest, servletResponse); - } + public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", + "proxy", "proxyPort", + "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", + "whitelist", + "blacklist", "prerenderServiceUrl", "protocol"); + private PrerenderSeoService prerenderSeoService; + + @Override + public void init(FilterConfig filterConfig) { + this.prerenderSeoService = new PrerenderSeoService(toMap(filterConfig)); + } + + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, + FilterChain filterChain) + throws IOException, ServletException { + boolean isPrerendered = prerenderSeoService.prerenderIfEligible( + (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); + if (!isPrerendered) { + filterChain.doFilter(servletRequest, servletResponse); } + } - @Override - public void destroy() { - prerenderSeoService.destroy(); - } + @Override + public void destroy() { + prerenderSeoService.destroy(); + } - protected void setPrerenderSeoService(PrerenderSeoService prerenderSeoService) { - this.prerenderSeoService = prerenderSeoService; - } + protected void setPrerenderSeoService(PrerenderSeoService prerenderSeoService) { + this.prerenderSeoService = prerenderSeoService; + } - protected Map toMap(FilterConfig filterConfig) { - Map config = Maps.newHashMap(); - for (String parameterName : PARAMETER_NAMES) { - config.put(parameterName, filterConfig.getInitParameter(parameterName)); - } - return config; + protected Map toMap(FilterConfig filterConfig) { + Map config = Maps.newHashMap(); + for (String parameterName : PARAMETER_NAMES) { + config.put(parameterName, filterConfig.getInitParameter(parameterName)); } + return config; + } } diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 900f2dd..ecb8a1a 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -1,7 +1,12 @@ package com.github.greengerong; -import com.google.common.collect.Lists; +import static org.apache.commons.lang.StringUtils.isNotBlank; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.apache.http.HttpHost; import org.apache.http.client.config.RequestConfig; @@ -11,126 +16,173 @@ import org.apache.http.impl.conn.DefaultProxyRoutePlanner; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.apache.commons.lang.StringUtils.isNotBlank; - @Slf4j public class PrerenderConfig { - public static final String PRERENDER_IO_SERVICE_URL = "https://service.prerender.io/"; - private final Map config; - public PrerenderConfig(Map config) { - this.config = config; + public static final String PRERENDER_IO_SERVICE_URL = "https://service.prerender.io/"; + private final Map config; + + private List crawlerUserAgents = List.of( + "baiduspider", + "facebookexternalhit", + "twitterbot", + "rogerbot", + "linkedinbot", + "embedly", + "quora link preview", + "showyoubot", + "outbrain", + "pinterest", + "developers.google.com/+/web/snippet", + "slackbot", + "vkShare", + "W3C_Validator", + "redditbot", + "Applebot", + "googlebot", + "yahoo! slurp", + "bingbot", + "yandex", + "whatsapp", + "flipboard", + "tumblr", + "bitlybot", + "skypeuripreview", + "nuzzel", + "discordbot", + "google page speed", + "qwantify", + "pinterestbot", + "bitrix link preview", + "xing-contenttabreceiver", + "chrome-lighthouse", + "telegrambot", + "Yeti", + "kakaotalk-scrap", + "Daumoa" + ); + private List extensionsToIgnore = List.of( + ".js", ".json", ".css", ".xml", ".less", ".png", ".jpg", + ".jpeg", ".gif", ".pdf", ".doc", ".txt", ".ico", ".rss", + ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", + ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", + ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", + ".iso", ".flv", ".m4v", ".torrent", ".woff", ".ttf" + ); + + public PrerenderConfig(Map config) { + this.config = config; + + // Initialize crawlerUserAgents + var customUAsStr = config.get("crawlerUserAgents"); + if (isNotBlank(customUAsStr)) { + crawlerUserAgents = Stream.concat( + crawlerUserAgents.stream(), + Arrays.stream(customUAsStr.trim().split(",")).map(String::trim) + ).toList(); } - public PreRenderEventHandler getEventHandler() { - final String preRenderEventHandler = config.get("preRenderEventHandler"); - if (isNotBlank(preRenderEventHandler)) { - try { - return (PreRenderEventHandler) Class.forName(preRenderEventHandler) - .getDeclaredConstructor().newInstance(); - } catch (Exception e) { - log.error("PreRenderEventHandler class not find or can not new a instance", e); - } - } - return null; + // Initialize extensionsToIgnore + var extToIgnoreStr = config.get("extensionsToIgnore"); + if (isNotBlank(extToIgnoreStr)) { + extensionsToIgnore = Stream.concat( + extensionsToIgnore.stream(), + Arrays.stream(extToIgnoreStr.trim().split(",")).map(String::trim) + ).toList(); } - - public CloseableHttpClient getHttpClient() { - HttpClientBuilder builder = HttpClients.custom() - .setConnectionManager(new PoolingHttpClientConnectionManager()) - .disableRedirectHandling(); - - configureProxy(builder); - configureTimeout(builder); - return builder.build(); + } + + public PreRenderEventHandler getEventHandler() { + final String preRenderEventHandler = config.get("preRenderEventHandler"); + if (isNotBlank(preRenderEventHandler)) { + try { + return (PreRenderEventHandler) Class.forName(preRenderEventHandler) + .getDeclaredConstructor().newInstance(); + } catch (Exception e) { + log.error("PreRenderEventHandler class not find or can not new a instance", e); + } } - - private void configureProxy(HttpClientBuilder builder) { - final String proxy = config.get("proxy"); - if (isNotBlank(proxy)) { - final int proxyPort = Integer.parseInt(config.get("proxyPort")); - DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort)); - builder.setRoutePlanner(routePlanner); - } - } - - private void configureTimeout(HttpClientBuilder builder) { - final String socketTimeout = getSocketTimeout(); - if (socketTimeout != null) { - RequestConfig config = RequestConfig.custom().setSocketTimeout(Integer.parseInt(socketTimeout)).build(); - builder.setDefaultRequestConfig(config); - } + return null; + } + + public CloseableHttpClient getHttpClient() { + HttpClientBuilder builder = HttpClients.custom() + .setConnectionManager(new PoolingHttpClientConnectionManager()) + .disableRedirectHandling(); + + configureProxy(builder); + configureTimeout(builder); + return builder.build(); + } + + private void configureProxy(HttpClientBuilder builder) { + final String proxy = config.get("proxy"); + if (isNotBlank(proxy)) { + final int proxyPort = Integer.parseInt(config.get("proxyPort")); + DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner( + new HttpHost(proxy, proxyPort)); + builder.setRoutePlanner(routePlanner); } - - public String getSocketTimeout() { - return config.get("socketTimeout"); - } - - public String getPrerenderToken() { - return config.get("prerenderToken"); + } + + private void configureTimeout(HttpClientBuilder builder) { + final String socketTimeout = getSocketTimeout(); + if (socketTimeout != null) { + RequestConfig config = RequestConfig.custom() + .setSocketTimeout(Integer.parseInt(socketTimeout)).build(); + builder.setDefaultRequestConfig(config); } + } - public String getForwardedURLHeader() { - return config.get("forwardedURLHeader"); - } + public String getSocketTimeout() { + return config.get("socketTimeout"); + } - public String getProtocol() { - return config.get("protocol"); - } + public String getPrerenderToken() { + return config.get("prerenderToken"); + } - public List getCrawlerUserAgents() { - List crawlerUserAgents = Lists.newArrayList("baiduspider", - "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview" - , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot", "vkShare", - "W3C_Validator", "redditbot", "Applebot"); - final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); - if (isNotBlank(crawlerUserAgentsFromConfig)) { - crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); - } - - return crawlerUserAgents; - } + public String getForwardedURLHeader() { + return config.get("forwardedURLHeader"); + } - public List getExtensionsToIgnore() { - List extensionsToIgnore = Lists.newArrayList(".js", ".json", ".css", ".xml", ".less", ".png", ".jpg", - ".jpeg", ".gif", ".pdf", ".doc", ".txt", ".ico", ".rss", ".zip", ".mp3", ".rar", ".exe", ".wmv", - ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", - ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent", ".woff", ".ttf"); - final String extensionsToIgnoreFromConfig = config.get("extensionsToIgnore"); - if (isNotBlank(extensionsToIgnoreFromConfig)) { - extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); - } - - return extensionsToIgnore; - } + public String getProtocol() { + return config.get("protocol"); + } - public List getWhitelist() { - final String whitelist = config.get("whitelist"); - if (isNotBlank(whitelist)) { - return Arrays.asList(whitelist.trim().split(",")); - } - return null; - } + public List getCrawlerUserAgents() { + return crawlerUserAgents; + } - public List getBlacklist() { - final String blacklist = config.get("blacklist"); - if (isNotBlank(blacklist)) { - return Arrays.asList(blacklist.trim().split(",")); - } - return null; - } + public List getExtensionsToIgnore() { + return extensionsToIgnore; + } - public String getPrerenderServiceUrl() { - final String prerenderServiceUrl = config.get("prerenderServiceUrl"); - return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : getDefaultPrerenderIoServiceUrl(); + public List getWhitelist() { + final String whitelist = config.get("whitelist"); + if (isNotBlank(whitelist)) { + return Arrays.asList(whitelist.trim().split(",")); } + return null; + } - private String getDefaultPrerenderIoServiceUrl() { - final String prerenderServiceUrlInEnv = System.getProperty("PRERENDER_SERVICE_URL"); - return isNotBlank(prerenderServiceUrlInEnv) ? prerenderServiceUrlInEnv : PRERENDER_IO_SERVICE_URL; + public List getBlacklist() { + final String blacklist = config.get("blacklist"); + if (isNotBlank(blacklist)) { + return Arrays.asList(blacklist.trim().split(",")); } + return null; + } + + public String getPrerenderServiceUrl() { + final String prerenderServiceUrl = config.get("prerenderServiceUrl"); + return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl + : getDefaultPrerenderIoServiceUrl(); + } + + private String getDefaultPrerenderIoServiceUrl() { + final String prerenderServiceUrlInEnv = System.getProperty("PRERENDER_SERVICE_URL"); + return isNotBlank(prerenderServiceUrlInEnv) ? prerenderServiceUrlInEnv + : PRERENDER_IO_SERVICE_URL; + } } diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index db17361..6bcd673 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -15,6 +15,7 @@ import java.util.regex.Pattern; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; import org.apache.http.Header; import org.apache.http.HeaderElement; @@ -31,326 +32,335 @@ import org.apache.http.message.BasicHeader; import org.apache.http.message.HeaderGroup; import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +@Slf4j public class PrerenderSeoService { - private final static Logger log = LoggerFactory.getLogger(PrerenderSeoService.class); - /** - * These are the "hop-by-hop" headers that should not be copied. - * ... - * I use an HttpClient HeaderGroup class instead of Set because this - * approach does case-insensitive lookup faster. - */ - private static final HeaderGroup hopByHopHeaders; - public static final String ESCAPED_FRAGMENT_KEY = "_escaped_fragment_"; - private final CloseableHttpClient httpClient; - private final PrerenderConfig prerenderConfig; - private PreRenderEventHandler preRenderEventHandler; - - public PrerenderSeoService(Map config) { - this.prerenderConfig = new PrerenderConfig(config); - this.httpClient = getHttpClient(); - } - static { - hopByHopHeaders = new HeaderGroup(); - String[] headers = new String[]{ - "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", - "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; - for (String header : headers) { - hopByHopHeaders.addHeader(new BasicHeader(header, null)); - } + /** + * These are the "hop-by-hop" headers that should not be copied. + * ... + * I use an HttpClient HeaderGroup class instead of Set because this approach does + * case-insensitive lookup faster. + */ + private static final HeaderGroup hopByHopHeaders; + public static final String ESCAPED_FRAGMENT_KEY = "_escaped_fragment_"; + private final CloseableHttpClient httpClient; + private final PrerenderConfig prerenderConfig; + private PreRenderEventHandler preRenderEventHandler; + + public PrerenderSeoService(Map config) { + this.prerenderConfig = new PrerenderConfig(config); + this.httpClient = getHttpClient(); + } + + static { + hopByHopHeaders = new HeaderGroup(); + String[] headers = new String[]{ + "Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", + "TE", "Trailers", "Transfer-Encoding", "Upgrade"}; + for (String header : headers) { + hopByHopHeaders.addHeader(new BasicHeader(header, null)); } + } - public void destroy() { - if (preRenderEventHandler != null) { - preRenderEventHandler.destroy(); - } - closeQuietly(httpClient); + public void destroy() { + if (preRenderEventHandler != null) { + preRenderEventHandler.destroy(); } + closeQuietly(httpClient); + } - public boolean prerenderIfEligible(HttpServletRequest servletRequest, HttpServletResponse servletResponse) { - try { - if (handlePrerender(servletRequest, servletResponse)) { - return true; - } - } catch (Exception e) { - log.error("Prerender service error", e); - } - return false; + public boolean prerenderIfEligible(HttpServletRequest servletRequest, + HttpServletResponse servletResponse) { + try { + if (handlePrerender(servletRequest, servletResponse)) { + return true; + } + } catch (Exception e) { + log.error("Prerender service error", e); } - - private boolean handlePrerender(HttpServletRequest servletRequest, HttpServletResponse servletResponse) - throws URISyntaxException, IOException { - if (shouldShowPrerenderedPage(servletRequest)) { - this.preRenderEventHandler = prerenderConfig.getEventHandler(); - return beforeRender(servletRequest, servletResponse) || proxyPrerenderedPageResponse( - servletRequest, servletResponse); - } - return false; + return false; + } + + private boolean handlePrerender(HttpServletRequest servletRequest, + HttpServletResponse servletResponse) + throws URISyntaxException, IOException { + if (shouldShowPrerenderedPage(servletRequest)) { + this.preRenderEventHandler = prerenderConfig.getEventHandler(); + return beforeRender(servletRequest, servletResponse) || proxyPrerenderedPageResponse( + servletRequest, servletResponse); } + return false; + } - private boolean shouldShowPrerenderedPage(HttpServletRequest request) { - final String userAgent = request.getHeader("User-Agent"); - final String url = getRequestURL(request); - final String referer = request.getHeader("Referer"); - - log.trace(String.format("checking request for %s from User-Agent %s and referer %s", url, userAgent, referer)); + private boolean shouldShowPrerenderedPage(HttpServletRequest request) { + final String userAgent = request.getHeader("User-Agent"); + final String url = getRequestURL(request); + final String referer = request.getHeader("Referer"); - if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { - log.trace("Request is not HTTP GET; intercept: no"); - return false; - } + log.trace( + String.format("checking request for %s from User-Agent %s and referer %s", url, userAgent, + referer)); - if (isInResources(url)) { - log.trace("request is for a (static) resource; intercept: no"); - return false; - } - - String prerenderHeader = request.getHeader("X-Prerender"); - if (StringUtils.isNotEmpty(prerenderHeader)) { - return false; - } - - final List whiteList = prerenderConfig.getWhitelist(); - if (whiteList != null && !isInWhiteList(url, whiteList)) { - log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); - return false; - } - - final List blacklist = prerenderConfig.getBlacklist(); - if (blacklist != null && isInBlackList(url, referer, blacklist)) { - log.trace("Blacklist is enabled, and this request is listed; intercept: no"); - return false; - } - - if (hasEscapedFragment(request)) { - log.trace("Request Has _escaped_fragment_; intercept: yes"); - return true; - } - - if (StringUtils.isBlank(userAgent)) { - log.trace("Request has blank userAgent; intercept: no"); - return false; - } + if (!HttpGet.METHOD_NAME.equals(request.getMethod())) { + log.trace("Request is not HTTP GET; intercept: no"); + return false; + } - if (!isInSearchUserAgent(userAgent)) { - log.trace("Request User-Agent is not a search bot; intercept: no"); - return false; - } + if (isInResources(url)) { + log.trace("request is for a (static) resource; intercept: no"); + return false; + } - log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); - return true; + String prerenderHeader = request.getHeader("X-Prerender"); + if (StringUtils.isNotEmpty(prerenderHeader)) { + return false; } - protected HttpGet getHttpGet(String apiUrl) { - return new HttpGet(apiUrl); + final List whiteList = prerenderConfig.getWhitelist(); + if (whiteList != null && !isInWhiteList(url, whiteList)) { + log.trace("Whitelist is enabled, but this request is not listed; intercept: no"); + return false; } - protected CloseableHttpClient getHttpClient() { - return prerenderConfig.getHttpClient(); + final List blacklist = prerenderConfig.getBlacklist(); + if (blacklist != null && isInBlackList(url, referer, blacklist)) { + log.trace("Blacklist is enabled, and this request is listed; intercept: no"); + return false; } - /** - * Copy request headers from the servlet client to the proxy request. - */ - private void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) - throws URISyntaxException { - // Get an Enumeration of all of the header names sent by the client - Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); - while (enumerationOfHeaderNames.hasMoreElements()) { - String headerName = (String) enumerationOfHeaderNames.nextElement(); - //Instead the content-length is effectively set via InputStreamEntity - if (!headerName.equalsIgnoreCase(CONTENT_LENGTH) && !hopByHopHeaders.containsHeader(headerName)) { - Enumeration headers = servletRequest.getHeaders(headerName); - while (headers.hasMoreElements()) {//sometimes more than one value - String headerValue = (String) headers.nextElement(); - // In case the proxy host is running multiple virtual servers, - // rewrite the Host header to ensure that we get content from - // the correct virtual server - if (headerName.equalsIgnoreCase(HOST)) { - HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); - headerValue = host.getHostName(); - if (host.getPort() != -1) { - headerValue += ":" + host.getPort(); - } - } - proxyRequest.addHeader(headerName, headerValue); - } - } - } + if (hasEscapedFragment(request)) { + log.trace("Request Has _escaped_fragment_; intercept: yes"); + return true; } - private String getRequestURL(HttpServletRequest request) { - if (StringUtils.isNotEmpty(prerenderConfig.getForwardedURLHeader())) { - String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); - if (StringUtils.isNotEmpty(url)) { - return url; - } - } - if (StringUtils.isNotEmpty(prerenderConfig.getProtocol())) { - var url = request.getRequestURL().toString(); - var schemeReplaced = url.replace(request.getScheme(), prerenderConfig.getProtocol()); - log.trace("getRequestURL: schemeReplaced: {} -> {}", url, schemeReplaced); - return schemeReplaced; - } - return request.getRequestURL().toString(); + if (StringUtils.isBlank(userAgent)) { + log.trace("Request has blank userAgent; intercept: no"); + return false; } - private String getApiUrl(String url) { - String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); - if (!prerenderServiceUrl.endsWith("/")) { - prerenderServiceUrl += "/"; - } - return prerenderServiceUrl + url; + if (!isInSearchUserAgent(userAgent)) { + log.trace("Request User-Agent is not a search bot; intercept: no"); + return false; } - /** - * Copy proxied response headers back to the servlet client. - */ - private void copyResponseHeaders(HttpResponse proxyResponse, final HttpServletResponse servletResponse) { - servletResponse.setCharacterEncoding(getContentCharSet(proxyResponse.getEntity())); - for (Header header : proxyResponse.getAllHeaders()) { - if (!hopByHopHeaders.containsHeader(header.getName())) { - servletResponse.addHeader(header.getName(), header.getValue()); + log.trace(String.format("Defaulting to request intercept(user-agent=%s): yes", userAgent)); + return true; + } + + protected HttpGet getHttpGet(String apiUrl) { + return new HttpGet(apiUrl); + } + + protected CloseableHttpClient getHttpClient() { + return prerenderConfig.getHttpClient(); + } + + /** + * Copy request headers from the servlet client to the proxy request. + */ + private void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) + throws URISyntaxException { + // Get an Enumeration of all of the header names sent by the client + Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames(); + while (enumerationOfHeaderNames.hasMoreElements()) { + String headerName = (String) enumerationOfHeaderNames.nextElement(); + //Instead the content-length is effectively set via InputStreamEntity + if (!headerName.equalsIgnoreCase(CONTENT_LENGTH) && !hopByHopHeaders.containsHeader( + headerName)) { + Enumeration headers = servletRequest.getHeaders(headerName); + while (headers.hasMoreElements()) {//sometimes more than one value + String headerValue = (String) headers.nextElement(); + // In case the proxy host is running multiple virtual servers, + // rewrite the Host header to ensure that we get content from + // the correct virtual server + if (headerName.equalsIgnoreCase(HOST)) { + HttpHost host = URIUtils.extractHost(new URI(prerenderConfig.getPrerenderServiceUrl())); + headerValue = host.getHostName(); + if (host.getPort() != -1) { + headerValue += ":" + host.getPort(); } + } + proxyRequest.addHeader(headerName, headerValue); } + } } - - /** - * Get the charset used to encode the http entity. - */ - private String getContentCharSet(final HttpEntity entity) throws ParseException { - if (entity == null) { - return null; - } - String charset = null; - if (entity.getContentType() != null) { - HeaderElement[] values = entity.getContentType().getElements(); - if (values.length > 0) { - NameValuePair param = values[0].getParameterByName("charset"); - if (param != null) { - charset = param.getValue(); - } - } - } - return charset; + } + + private String getRequestURL(HttpServletRequest request) { + if (StringUtils.isNotEmpty(prerenderConfig.getForwardedURLHeader())) { + String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); + if (StringUtils.isNotEmpty(url)) { + return url; + } } - - private String getResponseHtml(HttpResponse proxyResponse) - throws IOException { - HttpEntity entity = proxyResponse.getEntity(); - return entity != null ? EntityUtils.toString(entity) : ""; + if (StringUtils.isNotEmpty(prerenderConfig.getProtocol())) { + var url = request.getRequestURL().toString(); + var schemeReplaced = url.replace(request.getScheme(), prerenderConfig.getProtocol()); + log.trace("getRequestURL: schemeReplaced: {} -> {}", url, schemeReplaced); + return schemeReplaced; } + return request.getRequestURL().toString(); + } - /** - * Copy response body data (the entity) from the proxy to the servlet client. - */ - private void responseEntity(String html, HttpServletResponse servletResponse) - throws IOException { - PrintWriter printWriter = servletResponse.getWriter(); - try { - printWriter.write(html); - printWriter.flush(); - } finally { - closeQuietly(printWriter); - } + private String getApiUrl(String url) { + String prerenderServiceUrl = prerenderConfig.getPrerenderServiceUrl(); + if (!prerenderServiceUrl.endsWith("/")) { + prerenderServiceUrl += "/"; } - - - protected void closeQuietly(Closeable closeable) { - try { - if (closeable != null) { - closeable.close(); - } - } catch (IOException e) { - log.error("Close proxy error", e); - } + return prerenderServiceUrl + url; + } + + /** + * Copy proxied response headers back to the servlet client. + */ + private void copyResponseHeaders(HttpResponse proxyResponse, + final HttpServletResponse servletResponse) { + servletResponse.setCharacterEncoding(getContentCharSet(proxyResponse.getEntity())); + for (Header header : proxyResponse.getAllHeaders()) { + if (!hopByHopHeaders.containsHeader(header.getName())) { + servletResponse.addHeader(header.getName(), header.getValue()); + } } - - private boolean hasEscapedFragment(HttpServletRequest request) { - return request.getParameterMap().containsKey(ESCAPED_FRAGMENT_KEY); + } + + /** + * Get the charset used to encode the http entity. + */ + private String getContentCharSet(final HttpEntity entity) throws ParseException { + if (entity == null) { + return null; } - - private boolean isInBlackList(final String url, final String referer, List blacklist) { - return blacklist.stream().anyMatch(regex -> { - final Pattern pattern = Pattern.compile(regex); - return pattern.matcher(url).matches() || - (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); - }); + String charset = null; + if (entity.getContentType() != null) { + HeaderElement[] values = entity.getContentType().getElements(); + if (values.length > 0) { + NameValuePair param = values[0].getParameterByName("charset"); + if (param != null) { + charset = param.getValue(); + } + } } - - private boolean isInSearchUserAgent(final String userAgent) { - return prerenderConfig.getCrawlerUserAgents().stream() - .anyMatch(item -> userAgent.toLowerCase().contains(item.toLowerCase())); + return charset; + } + + private String getResponseHtml(HttpResponse proxyResponse) + throws IOException { + HttpEntity entity = proxyResponse.getEntity(); + return entity != null ? EntityUtils.toString(entity) : ""; + } + + /** + * Copy response body data (the entity) from the proxy to the servlet client. + */ + private void responseEntity(String html, HttpServletResponse servletResponse) + throws IOException { + PrintWriter printWriter = servletResponse.getWriter(); + try { + printWriter.write(html); + printWriter.flush(); + } finally { + closeQuietly(printWriter); } + } - private boolean isInResources(final String url) { - return prerenderConfig.getExtensionsToIgnore().stream() - .anyMatch(item -> (url.indexOf('?') >= 0 ? url.substring(0, url.indexOf('?')) : url) - .toLowerCase().endsWith(item)); + protected void closeQuietly(Closeable closeable) { + try { + if (closeable != null) { + closeable.close(); + } + } catch (IOException e) { + log.error("Close proxy error", e); } - - private boolean isInWhiteList(final String url, List whitelist) { - return whitelist.stream().anyMatch(regex -> Pattern.compile(regex).matcher(url).matches()); + } + + private boolean hasEscapedFragment(HttpServletRequest request) { + return request.getParameterMap().containsKey(ESCAPED_FRAGMENT_KEY); + } + + private boolean isInBlackList(final String url, final String referer, List blacklist) { + return blacklist.stream().anyMatch(regex -> { + final Pattern pattern = Pattern.compile(regex); + return pattern.matcher(url).matches() || + (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); + }); + } + + private boolean isInSearchUserAgent(final String userAgent) { + return prerenderConfig.getCrawlerUserAgents().stream() + .anyMatch(item -> userAgent.toLowerCase().contains(item.toLowerCase())); + } + + + private boolean isInResources(final String url) { + return prerenderConfig.getExtensionsToIgnore().stream() + .anyMatch(item -> (url.indexOf('?') >= 0 ? url.substring(0, url.indexOf('?')) : url) + .toLowerCase().endsWith(item)); + } + + private boolean isInWhiteList(final String url, List whitelist) { + return whitelist.stream().anyMatch(regex -> Pattern.compile(regex).matcher(url).matches()); + } + + private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) + throws IOException { + if (preRenderEventHandler != null) { + final String html = preRenderEventHandler.beforeRender(request); + if (isNotBlank(html)) { + final PrintWriter writer = response.getWriter(); + writer.write(html); + writer.flush(); + closeQuietly(writer); + return true; + } } - - private boolean beforeRender(HttpServletRequest request, HttpServletResponse response) throws IOException { - if (preRenderEventHandler != null) { - final String html = preRenderEventHandler.beforeRender(request); - if (isNotBlank(html)) { - final PrintWriter writer = response.getWriter(); - writer.write(html); - writer.flush(); - closeQuietly(writer); - return true; - } - } - return false; + return false; + } + + private boolean proxyPrerenderedPageResponse(HttpServletRequest request, + HttpServletResponse response) + throws IOException, URISyntaxException { + final String apiUrl = getApiUrl(getFullUrl(request)); + log.debug("Prerender proxy will send request to {}", apiUrl); + final HttpGet getMethod = getHttpGet(apiUrl); + copyRequestHeaders(request, getMethod); + withPrerenderToken(getMethod); + CloseableHttpResponse prerenderServerResponse = null; + + try { + prerenderServerResponse = httpClient.execute(getMethod); + response.setStatus(prerenderServerResponse.getStatusLine().getStatusCode()); + copyResponseHeaders(prerenderServerResponse, response); + String html = getResponseHtml(prerenderServerResponse); + html = afterRender(request, response, prerenderServerResponse, html); + responseEntity(html, response); + return true; + } finally { + closeQuietly(prerenderServerResponse); } + } - private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) - throws IOException, URISyntaxException { - final String apiUrl = getApiUrl(getFullUrl(request)); - log.debug("Prerender proxy will send request to {}", apiUrl); - final HttpGet getMethod = getHttpGet(apiUrl); - copyRequestHeaders(request, getMethod); - withPrerenderToken(getMethod); - CloseableHttpResponse prerenderServerResponse = null; - - try { - prerenderServerResponse = httpClient.execute(getMethod); - response.setStatus(prerenderServerResponse.getStatusLine().getStatusCode()); - copyResponseHeaders(prerenderServerResponse, response); - String html = getResponseHtml(prerenderServerResponse); - html = afterRender(request, response, prerenderServerResponse, html); - responseEntity(html, response); - return true; - } finally { - closeQuietly(prerenderServerResponse); - } + private String afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, + CloseableHttpResponse prerenderServerResponse, String responseHtml) { + if (preRenderEventHandler != null) { + return preRenderEventHandler.afterRender(clientRequest, clientResponse, + prerenderServerResponse, responseHtml); } - - private String afterRender(HttpServletRequest clientRequest, HttpServletResponse clientResponse, CloseableHttpResponse prerenderServerResponse, String responseHtml) { - if (preRenderEventHandler != null) { - return preRenderEventHandler.afterRender(clientRequest, clientResponse, prerenderServerResponse, responseHtml); - } - return responseHtml; - } - - private void withPrerenderToken(HttpRequest proxyRequest) { - final String token = prerenderConfig.getPrerenderToken(); - //for new version prerender with token. - if (isNotBlank(token)) { - proxyRequest.addHeader("X-Prerender-Token", token); - } + return responseHtml; + } + + private void withPrerenderToken(HttpRequest proxyRequest) { + final String token = prerenderConfig.getPrerenderToken(); + //for new version prerender with token. + if (isNotBlank(token)) { + proxyRequest.addHeader("X-Prerender-Token", token); } + } - private String getFullUrl(HttpServletRequest request) { - final String url = getRequestURL(request); - final String queryString = request.getQueryString(); - return isNotBlank(queryString) ? String.format("%s?%s", url, queryString) : url; - } + private String getFullUrl(HttpServletRequest request) { + final String url = getRequestURL(request); + final String queryString = request.getQueryString(); + return isNotBlank(queryString) ? String.format("%s?%s", url, queryString) : url; + } } diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 0d6e169..73e7051 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -32,340 +32,344 @@ @RunWith(MockitoJUnitRunner.class) public class PreRenderSEOFilterTest { - private PreRenderSEOFilter preRenderSEOFilter; - - @Mock - private CloseableHttpClient httpClient; - - @Mock - private FilterConfig filterConfig; - - @Mock - private HttpServletRequest servletRequest; - - @Mock - private HttpServletResponse servletResponse; - - @Mock - private FilterChain filterChain; - - @Mock - private HttpGet httpGet; - @Mock - private PrintWriter printWriter; - - @Before - public void setUp() { - preRenderSEOFilter = new PreRenderSEOFilter() { - @Override - public void init(FilterConfig filterConfig) { - setPrerenderSeoService(new PrerenderSeoService(toMap(filterConfig)) { - @Override - protected CloseableHttpClient getHttpClient() { - return httpClient; - } - - @Override - protected HttpGet getHttpGet(String apiUrl) { - return httpGet; - } - }); - } - }; - } - - @Test - public void should_not_handle_when_non_get_request() throws Exception { - //given - preRenderSEOFilter.init(filterConfig); - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer()); - when(servletRequest.getMethod()).thenReturn(HttpPost.METHOD_NAME); - - //when - preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); - - //then - verify(httpClient, never()).execute(httpGet); - verify(filterChain).doFilter(servletRequest, servletResponse); - } - - @Test - public void should_handle_when_protocol_set() throws Exception { - //given - when(filterConfig.getInitParameter("protocol")).thenReturn("https"); - preRenderSEOFilter.init(filterConfig); - final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); - final StatusLine statusLine = mock(StatusLine.class); - - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getScheme()).thenReturn("http"); - when(servletRequest.getMethod()).thenReturn(METHOD_NAME); - when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); - when(httpClient.execute(httpGet)).thenReturn(httpResponse); - when(httpResponse.getStatusLine()).thenReturn(statusLine); - final HashMap map = Maps.newHashMap(); - map.put(ESCAPED_FRAGMENT_KEY, ""); - when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(SC_OK); - when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); - when(servletResponse.getWriter()).thenReturn(printWriter); - - //when - preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); - - //then - verify(httpClient).execute(httpGet); - verify(filterChain, never()).doFilter(servletRequest, servletResponse); - } - - @Test - public void should_handle_when_url_with_escaped_fragment_() throws Exception { - //given - preRenderSEOFilter.init(filterConfig); - final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); - final StatusLine statusLine = mock(StatusLine.class); - - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(METHOD_NAME); - when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); - when(httpClient.execute(httpGet)).thenReturn(httpResponse); - when(httpResponse.getStatusLine()).thenReturn(statusLine); - final HashMap map = Maps.newHashMap(); - map.put(ESCAPED_FRAGMENT_KEY, ""); - when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(SC_OK); - when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); - when(servletResponse.getWriter()).thenReturn(printWriter); - - //when - preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); - - //then - verify(httpClient).execute(httpGet); - verify(filterChain, never()).doFilter(servletRequest, servletResponse); - } - - @Test - public void should_not_handle_when_user_agent_is_not_crawler() throws Exception { - //given - preRenderSEOFilter.init(filterConfig); - - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(METHOD_NAME); - when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); - when(servletRequest.getHeader("User-Agent")).thenReturn("no"); - //when - preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); - - //then - verify(httpClient, never()).execute(httpGet); - verify(filterChain).doFilter(servletRequest, servletResponse); - } - - @Test - public void should_not_handle_when_x_prerender_header_is_present() throws Exception { - //given - when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); - preRenderSEOFilter.init(filterConfig); - - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(METHOD_NAME); - when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); - when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); - when(servletRequest.getHeader("X-Prerender")).thenReturn("1"); - //when - preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); - - //then - verify(httpClient, never()).execute(httpGet); - verify(filterChain).doFilter(servletRequest, servletResponse); - } - - @Test - public void should_not_handle_when_url_is_a_resource() throws Exception { - //given - when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); - preRenderSEOFilter.init(filterConfig); - - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test.js")); - when(servletRequest.getMethod()).thenReturn(METHOD_NAME); - when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); - when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); - //when - preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); - - //then - verify(httpClient, never()).execute(httpGet); - verify(filterChain).doFilter(servletRequest, servletResponse); - } - - @Test - public void should_not_handle_when_white_list_is_not_empty_and_url_is_not_in_white_list() throws Exception { - //given - when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); - when(filterConfig.getInitParameter("whitelist")).thenReturn("whitelist1,whitelist2"); - preRenderSEOFilter.init(filterConfig); - - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(METHOD_NAME); - when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); - when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); - //when - preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); - - //then - verify(httpClient, never()).execute(httpGet); - verify(filterChain).doFilter(servletRequest, servletResponse); - } - - @Test - public void should_not_handle_when_black_list_is_not_empty_and_url_is_in_black_list() throws Exception { - //given - when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); - when(filterConfig.getInitParameter("blacklist")).thenReturn("blacklist1,http://localhost/test"); - preRenderSEOFilter.init(filterConfig); - - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(METHOD_NAME); - when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); - when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); - //when - preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); - - //then - verify(httpClient, never()).execute(httpGet); - verify(filterChain).doFilter(servletRequest, servletResponse); - } - - @Test - public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and_white_list_is_empty_and_black_list_is_empty() throws Exception { - //given - when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); - preRenderSEOFilter.init(filterConfig); - - final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); - final StatusLine statusLine = mock(StatusLine.class); - - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(METHOD_NAME); - when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); - - when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); - when(httpClient.execute(httpGet)).thenReturn(httpResponse); - when(httpResponse.getStatusLine()).thenReturn(statusLine); - final HashMap map = Maps.newHashMap(); - map.put(ESCAPED_FRAGMENT_KEY, ""); - when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(SC_OK); - when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); - when(servletResponse.getWriter()).thenReturn(printWriter); - //when - preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); - - //then - verify(httpClient).execute(httpGet); - verify(filterChain, never()).doFilter(servletRequest, servletResponse); - } - - @Test - public void should_handle_when_every_thing_is_ok_but_prerender_server_response_is_not_200() throws Exception { - //given - when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); - preRenderSEOFilter.init(filterConfig); - - final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); - final StatusLine statusLine = mock(StatusLine.class); - - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(METHOD_NAME); - when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); - - when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); - when(httpClient.execute(httpGet)).thenReturn(httpResponse); - when(httpResponse.getStatusLine()).thenReturn(statusLine); - final HashMap map = Maps.newHashMap(); - map.put(ESCAPED_FRAGMENT_KEY, ""); - when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(SC_NOT_FOUND); - when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); - when(servletResponse.getWriter()).thenReturn(printWriter); - - - //when - preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); - - //then - verify(httpClient).execute(httpGet); - verify(filterChain, never()).doFilter(servletRequest, servletResponse); - verify(servletResponse).setStatus(SC_NOT_FOUND); - } - - - @Test - public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and_in_white_list_and_not_in_black_list() throws Exception { - //given - when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); - when(filterConfig.getInitParameter("whitelist")).thenReturn("whitelist1,http://localhost/test"); - when(filterConfig.getInitParameter("blacklist")).thenReturn("blacklist1,blacklist2"); - - preRenderSEOFilter.init(filterConfig); - - final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); - final StatusLine statusLine = mock(StatusLine.class); - - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(METHOD_NAME); - when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); - - when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); - when(httpClient.execute(httpGet)).thenReturn(httpResponse); - when(httpResponse.getStatusLine()).thenReturn(statusLine); - final HashMap map = Maps.newHashMap(); - map.put(ESCAPED_FRAGMENT_KEY, ""); - when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(SC_OK); - when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); - when(servletResponse.getWriter()).thenReturn(printWriter); - - //when - preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); - - //then - verify(httpClient).execute(httpGet); - verify(filterChain, never()).doFilter(servletRequest, servletResponse); - } - - @Test - public void should_use_request_url_from_custom_header_if_available() throws Exception { - //given - when(filterConfig.getInitParameter("forwardedURLHeader")).thenReturn("X-Forwarded-URL"); - when(filterConfig.getInitParameter("whitelist")).thenReturn("http://my.public.domain.com/"); - when(filterConfig.getInitParameter("blacklist")).thenReturn("http://localhost/test"); - - preRenderSEOFilter.init(filterConfig); - - final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); - final StatusLine statusLine = mock(StatusLine.class); - - when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); - when(servletRequest.getMethod()).thenReturn(METHOD_NAME); - when(servletRequest.getHeader("X-Forwarded-URL")).thenReturn("http://my.public.domain.com/"); - - when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); - when(httpClient.execute(httpGet)).thenReturn(httpResponse); - when(httpResponse.getStatusLine()).thenReturn(statusLine); - final HashMap map = Maps.newHashMap(); - map.put(ESCAPED_FRAGMENT_KEY, ""); - when(servletRequest.getParameterMap()).thenReturn(map); - when(statusLine.getStatusCode()).thenReturn(SC_OK); - when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); - when(servletResponse.getWriter()).thenReturn(printWriter); - - //when - preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); - - //then - verify(httpClient).execute(httpGet); - verify(filterChain, never()).doFilter(servletRequest, servletResponse); - } + private PreRenderSEOFilter preRenderSEOFilter; + + @Mock + private CloseableHttpClient httpClient; + + @Mock + private FilterConfig filterConfig; + + @Mock + private HttpServletRequest servletRequest; + + @Mock + private HttpServletResponse servletResponse; + + @Mock + private FilterChain filterChain; + + @Mock + private HttpGet httpGet; + @Mock + private PrintWriter printWriter; + + @Before + public void setUp() { + preRenderSEOFilter = new PreRenderSEOFilter() { + @Override + public void init(FilterConfig filterConfig) { + setPrerenderSeoService(new PrerenderSeoService(toMap(filterConfig)) { + @Override + protected CloseableHttpClient getHttpClient() { + return httpClient; + } + + @Override + protected HttpGet getHttpGet(String apiUrl) { + return httpGet; + } + }); + } + }; + } + + @Test + public void should_not_handle_when_non_get_request() throws Exception { + //given + preRenderSEOFilter.init(filterConfig); + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer()); + when(servletRequest.getMethod()).thenReturn(HttpPost.METHOD_NAME); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_handle_when_protocol_set() throws Exception { + //given + when(filterConfig.getInitParameter("protocol")).thenReturn("https"); + preRenderSEOFilter.init(filterConfig); + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getScheme()).thenReturn("http"); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put(ESCAPED_FRAGMENT_KEY, ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(SC_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_handle_when_url_with_escaped_fragment_() throws Exception { + //given + preRenderSEOFilter.init(filterConfig); + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put(ESCAPED_FRAGMENT_KEY, ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(SC_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_user_agent_is_not_crawler() throws Exception { + //given + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("no"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_x_prerender_header_is_present() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + when(servletRequest.getHeader("X-Prerender")).thenReturn("1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_url_is_a_resource() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test.js")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_white_list_is_not_empty_and_url_is_not_in_white_list() + throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + when(filterConfig.getInitParameter("whitelist")).thenReturn("whitelist1,whitelist2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_not_handle_when_black_list_is_not_empty_and_url_is_in_black_list() + throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + when(filterConfig.getInitParameter("blacklist")).thenReturn("blacklist1,http://localhost/test"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and_white_list_is_empty_and_black_list_is_empty() + throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put(ESCAPED_FRAGMENT_KEY, ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(SC_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_handle_when_every_thing_is_ok_but_prerender_server_response_is_not_200() + throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put(ESCAPED_FRAGMENT_KEY, ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(SC_NOT_FOUND); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + verify(servletResponse).setStatus(SC_NOT_FOUND); + } + + + @Test + public void should_handle_when_user_agent_is_crawler_and_url_is_not_resource_and_in_white_list_and_not_in_black_list() + throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + when(filterConfig.getInitParameter("whitelist")).thenReturn("whitelist1,http://localhost/test"); + when(filterConfig.getInitParameter("blacklist")).thenReturn("blacklist1,blacklist2"); + + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put(ESCAPED_FRAGMENT_KEY, ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(SC_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } + + @Test + public void should_use_request_url_from_custom_header_if_available() throws Exception { + //given + when(filterConfig.getInitParameter("forwardedURLHeader")).thenReturn("X-Forwarded-URL"); + when(filterConfig.getInitParameter("whitelist")).thenReturn("http://my.public.domain.com/"); + when(filterConfig.getInitParameter("blacklist")).thenReturn("http://localhost/test"); + + preRenderSEOFilter.init(filterConfig); + + final CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + final StatusLine statusLine = mock(StatusLine.class); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getHeader("X-Forwarded-URL")).thenReturn("http://my.public.domain.com/"); + + when(servletRequest.getHeaderNames()).thenReturn(mock(Enumeration.class)); + when(httpClient.execute(httpGet)).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + final HashMap map = Maps.newHashMap(); + map.put(ESCAPED_FRAGMENT_KEY, ""); + when(servletRequest.getParameterMap()).thenReturn(map); + when(statusLine.getStatusCode()).thenReturn(SC_OK); + when(httpResponse.getAllHeaders()).thenReturn(new Header[0]); + when(servletResponse.getWriter()).thenReturn(printWriter); + + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient).execute(httpGet); + verify(filterChain, never()).doFilter(servletRequest, servletResponse); + } } diff --git a/src/test/java/com/github/greengerong/PrerenderConfigTest.java b/src/test/java/com/github/greengerong/PrerenderConfigTest.java index 58f5641..c4be2ef 100644 --- a/src/test/java/com/github/greengerong/PrerenderConfigTest.java +++ b/src/test/java/com/github/greengerong/PrerenderConfigTest.java @@ -8,25 +8,26 @@ import org.junit.Test; public class PrerenderConfigTest { - @Test(expected = Exception.class) - public void should_throw_exception_if_invalid_timeout_value_specified() { - //given - Map configuration = Maps.newHashMap(); - configuration.put("socketTimeout", "not_an_int"); - PrerenderConfig config = new PrerenderConfig(configuration); - //when - config.getHttpClient(); - } - @Test - public void should_pass_if_correct_timeout_value_specified() { - //given - Map configuration = Maps.newHashMap(); - configuration.put("socketTimeout", "1000"); - PrerenderConfig config = new PrerenderConfig(configuration); - //when - final CloseableHttpClient httpClient = config.getHttpClient(); + @Test(expected = Exception.class) + public void should_throw_exception_if_invalid_timeout_value_specified() { + //given + Map configuration = Maps.newHashMap(); + configuration.put("socketTimeout", "not_an_int"); + PrerenderConfig config = new PrerenderConfig(configuration); + //when + config.getHttpClient(); + } - assertNotNull(httpClient); - } + @Test + public void should_pass_if_correct_timeout_value_specified() { + //given + Map configuration = Maps.newHashMap(); + configuration.put("socketTimeout", "1000"); + PrerenderConfig config = new PrerenderConfig(configuration); + //when + final CloseableHttpClient httpClient = config.getHttpClient(); + + assertNotNull(httpClient); + } }