From c62cba8dfb5888d489c724a672a8a86ccea6341a Mon Sep 17 00:00:00 2001 From: Alex Edwards Date: Sun, 15 Dec 2013 20:12:56 -0800 Subject: [PATCH 1/2] Updated guava dependency As a result of scanning bug with weld in jboss presence of beans.xml is detected incorrectly. --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index f506089..7d5ee32 100644 --- a/pom.xml +++ b/pom.xml @@ -49,6 +49,7 @@ com.google.guava guava ${guava.version} + cdi1.0 junit From f05946321145903f5415cebedd0a7f713bfcb8f9 Mon Sep 17 00:00:00 2001 From: Alex Edwards Date: Sat, 28 Dec 2013 13:04:28 -0800 Subject: [PATCH 2/2] Refactored the main filter class, primarily to make it easier to test. Created constants enum to contain fitler parameters and defaults, created config class that deals with config initialization stuff. Added tests. Some refactoring in shouldShowPrerenderedResult because this is part of the blocking path and most requests will not be prerendered we want to fail fast with minimum work. --- pom.xml | 23 +- prerender-java.iml | 6 +- .../greengerong/PreRenderSEOFilter.java | 206 ++++-------------- .../com/github/greengerong/RenderUtils.java | 71 ++++++ .../github/greengerong/ResponseResult.java | 25 +++ .../greengerong/config/FilterParameter.java | 34 +++ .../greengerong/config/SeoFilterConfig.java | 120 ++++++++++ .../github/greengerong/RenderUtilsTest.java | 177 +++++++++++++++ 8 files changed, 492 insertions(+), 170 deletions(-) create mode 100644 src/main/java/com/github/greengerong/RenderUtils.java create mode 100644 src/main/java/com/github/greengerong/ResponseResult.java create mode 100644 src/main/java/com/github/greengerong/config/FilterParameter.java create mode 100644 src/main/java/com/github/greengerong/config/SeoFilterConfig.java create mode 100644 src/test/java/com/github/greengerong/RenderUtilsTest.java diff --git a/pom.xml b/pom.xml index 7d5ee32..7e7a0d0 100644 --- a/pom.xml +++ b/pom.xml @@ -30,6 +30,7 @@ UTF-8 4.11 + 3.2 1.9.5 15.0 2.6 @@ -39,12 +40,6 @@ - - junit - junit - 4.11 - test - com.google.guava guava @@ -57,6 +52,12 @@ ${junit.version} test + + org.easymock + easymock + ${easymock.version} + test + org.mockito mockito-all @@ -79,11 +80,11 @@ commons-httpclient ${commons-httpclient.version} - - - - - + + log4j + log4j + ${log4j.version} + diff --git a/prerender-java.iml b/prerender-java.iml index 2dc2562..2ff1f47 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -10,15 +10,19 @@ + - + + + + diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 5e9defd..cd7fc62 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,57 +1,66 @@ package com.github.greengerong; -import com.google.common.base.Predicate; -import com.google.common.collect.Lists; +import com.github.greengerong.config.SeoFilterConfig; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; import javax.servlet.*; import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.io.PrintWriter; -import java.net.URISyntaxException; -import java.util.Arrays; import java.util.List; -import java.util.regex.Pattern; - -import static com.google.common.collect.FluentIterable.from; public class PreRenderSEOFilter implements Filter { + public static final String REFERER_HEADER = "Referer"; + private static final String FORWARD_SLASH = "/"; + + public static Logger LOG = Logger.getLogger(PreRenderSEOFilter.class); - private FilterConfig filterConfig; + private SeoFilterConfig config; @Override public void init(FilterConfig filterConfig) throws ServletException { - this.filterConfig = filterConfig; + config = new SeoFilterConfig(filterConfig); + LOG.info("Initialization Complete"); } @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - try { - final HttpServletRequest request = (HttpServletRequest) servletRequest; - if (shouldShowPrerenderedPage(request)) { + + final HttpServletRequest request = (HttpServletRequest) servletRequest; + if (shouldShowPrerenderedPage(request)) { + try { final ResponseResult result = getPrerenderedPageResponse(request); - if (result.getStatusCode() == 200) { + if (result.getStatusCode() == HttpStatus.SC_OK) { final PrintWriter writer = servletResponse.getWriter(); writer.write(result.getResponseBody()); writer.flush(); - return; } + } catch (Exception e) { + LOG.warn("An error occured processing the request", e); } - } catch (Exception e) { + }else { + filterChain.doFilter(servletRequest, servletResponse); } - filterChain.doFilter(servletRequest, servletResponse); + } + + @Override + public void destroy() { + config = null; + LOG.info("Shutting down"); } private ResponseResult getPrerenderedPageResponse(HttpServletRequest request) throws IOException { - final String apiUrl = getApiUrl(getFullUrl(request)); + final String apiUrl = getApiUrl(request, config); final HttpClient httpClient = new HttpClient(); final GetMethod getMethod = new GetMethod(apiUrl); - setConfig(httpClient); + setProxy(httpClient); setHttpHeader(getMethod); final int code = httpClient.executeMethod(getMethod); final String responseBody = getMethod.getResponseBodyAsString(); @@ -59,7 +68,15 @@ private ResponseResult getPrerenderedPageResponse(HttpServletRequest request) th return new ResponseResult(code, body); } - private String getFullUrl(HttpServletRequest request) { + public static String getApiUrl(HttpServletRequest request, SeoFilterConfig config) { + String prerenderServiceUrl = config.getPrerenderServiceUrl(); + if (!prerenderServiceUrl.endsWith(FORWARD_SLASH)) { + prerenderServiceUrl += FORWARD_SLASH; + } + return prerenderServiceUrl + getFullUrl(request); + } + + private static String getFullUrl(HttpServletRequest request) { final StringBuffer url = request.getRequestURL(); final String queryString = request.getQueryString(); if (queryString != null) { @@ -74,164 +91,37 @@ private void setHttpHeader(HttpMethod httpMethod) { httpMethod.setRequestHeader("Content-Type", "text/html"); } - private void setConfig(HttpClient httpClient) { - final String proxy = filterConfig.getInitParameter("proxy"); - if (StringUtils.isNotBlank(proxy)) { - final int proxyPort = Integer.parseInt(filterConfig.getInitParameter("proxyPort")); - httpClient.getHostConfiguration().setProxy(proxy, proxyPort); - } - } - - @Override - public void destroy() { - filterConfig = null; - } - - private List getCrawlerUserAgents() { - List crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider", - "facebookexternalhit", "twitterbot"); - final String crawlerUserAgentsFromConfig = filterConfig.getInitParameter("crawlerUserAgents"); - if (StringUtils.isNotBlank(crawlerUserAgentsFromConfig)) { - crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); - } - - return crawlerUserAgents; - } - - private List getExtensionsToIgnore() { - List extensionsToIgnore = Lists.newArrayList(".js", ".css", ".less", ".png", ".jpg", ".jpeg", - ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", - ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", - ".iso", ".flv", ".m4v", ".torrent"); - final String extensionsToIgnoreFromConfig = filterConfig.getInitParameter("extensionsToIgnore"); - if (StringUtils.isNotBlank(extensionsToIgnoreFromConfig)) { - extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); + private void setProxy(HttpClient httpClient) { + if (StringUtils.isNotBlank(config.getProxy())) { + httpClient.getHostConfiguration().setProxy(config.getProxy(), config.getProxyPort()); } - - return extensionsToIgnore; } - private List getWhitelist() { - final String whitelist = filterConfig.getInitParameter("whitelist"); - if (StringUtils.isNotBlank(whitelist)) { - return Arrays.asList(whitelist.trim().split(",")); - } - return null; - } - - private List getBlacklist() { - final String blacklist = filterConfig.getInitParameter("blacklist"); - if (StringUtils.isNotBlank(blacklist)) { - return Arrays.asList(blacklist.trim().split(",")); - } - return null; - } - - private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URISyntaxException { - final String useAgent = request.getHeader("User-Agent"); - final String url = request.getRequestURL().toString(); - final String referer = request.getHeader("Referer"); - - if (hasEscapedFragment(request)) { + private boolean shouldShowPrerenderedPage(HttpServletRequest request){ + if (RenderUtils.hasEscapedFragment(request)) { return true; } - if (StringUtils.isBlank(useAgent)) { - return false; - } - - if (!isInSearchUserAgent(useAgent)) { + if (!RenderUtils.isValidUserAgent(request, config)) { return false; } + final String url = request.getRequestURI(); - if (isInResources(url)) { + if (RenderUtils.ignoreFileExtension(url, config)) { return false; } - final List whiteList = getWhitelist(); - if (whiteList != null && !isInWhiteList(url, whiteList)) { + final List whiteList = config.getWhitelist(); + if (!whiteList.isEmpty() && !RenderUtils.isInWhiteList(url, whiteList)) { return false; } - final List blacklist = getBlacklist(); - if (blacklist != null && isInBlackList(url, referer, blacklist)) { + final List blacklist = config.getBlacklist(); + if (!blacklist.isEmpty() && RenderUtils.isInBlackList(url, request.getHeader(REFERER_HEADER), blacklist)) { return false; } return true; } - - private boolean hasEscapedFragment(HttpServletRequest request) { - return request.getParameterMap().containsKey("_escaped_fragment_"); - } - - private String getApiUrl(String url) { - String prerenderServiceUrl = getPrerenderServiceUrl(); - if (!prerenderServiceUrl.endsWith("/")) { - prerenderServiceUrl += "/"; - } - return prerenderServiceUrl + url; - } - - private String getPrerenderServiceUrl() { - final String prerenderServiceUrl = filterConfig.getInitParameter("prerenderServiceUrl"); - return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : "http://prerender.herokuapp.com/"; - } - - private boolean isInBlackList(final String url, final String referer, List blacklist) { - return from(blacklist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - final Pattern pattern = Pattern.compile(regex); - return pattern.matcher(url).matches() || - (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); - } - }); - } - - private boolean isInWhiteList(final String url, List whitelist) { - return from(whitelist).anyMatch(new Predicate() { - @Override - public boolean apply(String regex) { - return Pattern.compile(regex).matcher(url).matches(); - } - }); - } - - private boolean isInResources(final String url) { - return from(getExtensionsToIgnore()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return url.contains(item.toLowerCase()); - } - }); - } - - private boolean isInSearchUserAgent(final String useAgent) { - return from(getCrawlerUserAgents()).anyMatch(new Predicate() { - @Override - public boolean apply(String item) { - return item.equalsIgnoreCase(useAgent); - } - }); - } - - private class ResponseResult { - private int statusCode; - private String responseBody; - - public ResponseResult(int code, String body) { - statusCode = code; - responseBody = body; - } - - private int getStatusCode() { - return statusCode; - } - - private String getResponseBody() { - return responseBody; - } - } } diff --git a/src/main/java/com/github/greengerong/RenderUtils.java b/src/main/java/com/github/greengerong/RenderUtils.java new file mode 100644 index 0000000..24f0295 --- /dev/null +++ b/src/main/java/com/github/greengerong/RenderUtils.java @@ -0,0 +1,71 @@ +package com.github.greengerong; + +import com.github.greengerong.config.SeoFilterConfig; +import com.google.common.base.Predicate; +import org.apache.commons.lang.StringUtils; + +import javax.servlet.http.HttpServletRequest; +import java.util.List; +import java.util.regex.Pattern; + +import static com.google.common.collect.FluentIterable.from; + +/** + * Created with IntelliJ IDEA. + * User: set321go + * Date: 12/27/2013 + * Time: 10:04 PM + */ +public class RenderUtils { + public static final String ESCAPED_FRAGMENT_PARAMETER = "_escaped_fragment_"; + public static final String USER_AGENT_HEADER = "User-Agent"; + + + public static boolean hasEscapedFragment(HttpServletRequest request) { + return request.getParameterMap().containsKey(ESCAPED_FRAGMENT_PARAMETER); + } + + public static boolean isInBlackList(final String url, final String referer, List blacklist) { + return from(blacklist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + final Pattern pattern = Pattern.compile(regex); + return pattern.matcher(url).matches() || + (!StringUtils.isBlank(referer) && pattern.matcher(referer).matches()); + } + }); + } + + public static boolean isInWhiteList(final String url, List whitelist) { + return from(whitelist).anyMatch(new Predicate() { + @Override + public boolean apply(String regex) { + return Pattern.compile(regex).matcher(url).matches(); + } + }); + } + + public static boolean ignoreFileExtension(final String url, SeoFilterConfig config) { + return from(config.getExtensions()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return url.contains(item.toLowerCase()); + } + }); + } + + public static boolean isValidUserAgent(HttpServletRequest request, SeoFilterConfig config) { + final String useAgent = request.getHeader(USER_AGENT_HEADER); + + if (StringUtils.isBlank(useAgent)) { + return false; + }else{ + return from(config.getCrawlers()).anyMatch(new Predicate() { + @Override + public boolean apply(String item) { + return item.equalsIgnoreCase(useAgent); + } + }); + } + } +} diff --git a/src/main/java/com/github/greengerong/ResponseResult.java b/src/main/java/com/github/greengerong/ResponseResult.java new file mode 100644 index 0000000..e2d569e --- /dev/null +++ b/src/main/java/com/github/greengerong/ResponseResult.java @@ -0,0 +1,25 @@ +package com.github.greengerong; + +/** + * Created with IntelliJ IDEA. + * User: set321go + * Date: 12/27/2013 + * Time: 9:56 PM + */ +public class ResponseResult { + private final int statusCode; + private final String responseBody; + + public ResponseResult(int code, String body) { + statusCode = code; + responseBody = body; + } + + public int getStatusCode() { + return statusCode; + } + + public String getResponseBody() { + return responseBody; + } +} diff --git a/src/main/java/com/github/greengerong/config/FilterParameter.java b/src/main/java/com/github/greengerong/config/FilterParameter.java new file mode 100644 index 0000000..5d81311 --- /dev/null +++ b/src/main/java/com/github/greengerong/config/FilterParameter.java @@ -0,0 +1,34 @@ +package com.github.greengerong.config; + +/** + * Created with IntelliJ IDEA. + * User: set321go + * Date: 12/26/2013 + * Time: 11:26 PM + */ +public enum FilterParameter { + CRAWLER_USER_AGENTS("crawlerUserAgents", "googlebot,yahoo,bingbot,baiduspider,facebookexternalhit,twitterbot"), + EXTENSIONS_TO_IGNORE("extensionsToIgnore", ".js,.css,.less,.png,.jpg,.jpeg,.gif,.pdf,.doc,.txt,.zip,.mp3,.rar," + + ".exe,.wmv,.doc,.avi,.ppt,.mpg,.mpeg,.tif,.wav,.mov,.psd,.ai,.xls,.mp4,.m4a,.swf,.dat,.dmg,.iso,.flv,.m4v,.torrent"), + WHITELIST("whitelist", ""), + BLACKLIST("blacklist", ""), + SERVICE_URL("prerenderServiceUrl", "http://prerender.herokuapp.com/"), + PROXY_HOST("proxy", ""), + PROXY_PORT("proxyPort", "80"); + + private String paramName; + private String defaultValue; + + private FilterParameter(String paramName, String defaultValue){ + this.paramName = paramName; + this.defaultValue = defaultValue; + } + + public String getName(){ + return paramName; + } + + public String getDefaultValue(){ + return defaultValue; + } +} diff --git a/src/main/java/com/github/greengerong/config/SeoFilterConfig.java b/src/main/java/com/github/greengerong/config/SeoFilterConfig.java new file mode 100644 index 0000000..a0c84ec --- /dev/null +++ b/src/main/java/com/github/greengerong/config/SeoFilterConfig.java @@ -0,0 +1,120 @@ +package com.github.greengerong.config; + +import com.google.common.collect.Lists; +import org.apache.commons.lang.StringUtils; + +import javax.servlet.FilterConfig; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Created with IntelliJ IDEA. + * User: set321go + * Date: 12/26/2013 + * Time: 11:17 PM + */ +public class SeoFilterConfig { + public static final String LIST_SEPERATOR = ","; + + private final String prerenderServiceUrl; + private final List blacklist; + private final List whitelist; + private final List extensions; + private final List crawlers; + private final String proxy; + private final int proxyPort; + + public SeoFilterConfig(FilterConfig filterConfig){ + crawlers = loadCrawlerUserAgents(filterConfig); + extensions = loadExtensionsToIgnore(filterConfig); + whitelist = loadWhitelist(filterConfig); + blacklist = loadBlacklist(filterConfig); + prerenderServiceUrl = loadPrerenderServiceUrl(filterConfig); + proxy = loadProxyHost(filterConfig); + proxyPort = loadProxyPort(filterConfig); + } + + private List loadCrawlerUserAgents(FilterConfig filterConfig) { + List crawlerUserAgents = Lists.newArrayList(FilterParameter.CRAWLER_USER_AGENTS.getDefaultValue().split(LIST_SEPERATOR)); + final String crawlerUserAgentsFromConfig = filterConfig.getInitParameter(FilterParameter.CRAWLER_USER_AGENTS.getName()); + if (StringUtils.isNotBlank(crawlerUserAgentsFromConfig)) { + crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(LIST_SEPERATOR))); + } + + return crawlerUserAgents; + } + + private List loadExtensionsToIgnore(FilterConfig filterConfig) { + List extensionsToIgnore = Lists.newArrayList(FilterParameter.EXTENSIONS_TO_IGNORE.getDefaultValue().split(LIST_SEPERATOR)); + final String extensionsToIgnoreFromConfig = filterConfig.getInitParameter(FilterParameter.EXTENSIONS_TO_IGNORE.getName()); + if (StringUtils.isNotBlank(extensionsToIgnoreFromConfig)) { + extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(LIST_SEPERATOR))); + } + + return extensionsToIgnore; + } + + private String loadPrerenderServiceUrl(FilterConfig filterConfig) { + final String prerenderServiceUrl = filterConfig.getInitParameter(FilterParameter.SERVICE_URL.getName()); + return StringUtils.isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : FilterParameter.SERVICE_URL.getDefaultValue(); + } + + private List loadWhitelist(FilterConfig filterConfig) { + final String whitelist = filterConfig.getInitParameter(FilterParameter.WHITELIST.getName()); + if (StringUtils.isNotBlank(whitelist)) { + return Arrays.asList(whitelist.trim().split(LIST_SEPERATOR)); + } + return Collections.emptyList(); + } + + private List loadBlacklist(FilterConfig filterConfig) { + final String blacklist = filterConfig.getInitParameter(FilterParameter.BLACKLIST.getName()); + if (StringUtils.isNotBlank(blacklist)) { + return Arrays.asList(blacklist.trim().split(LIST_SEPERATOR)); + } + return Collections.emptyList(); + } + + private String loadProxyHost(FilterConfig filterConfig){ + final String proxy = filterConfig.getInitParameter(FilterParameter.PROXY_HOST.getName()); + return StringUtils.isNotBlank(proxy) ? proxy : FilterParameter.PROXY_HOST.getDefaultValue(); + } + + private int loadProxyPort(FilterConfig filterConfig){ + final String proxyPort = filterConfig.getInitParameter(FilterParameter.PROXY_PORT.getName()); + if (StringUtils.isNotBlank(proxy) && StringUtils.isNotBlank(proxyPort)) { + return Integer.parseInt(proxyPort); + }else { + return Integer.parseInt(FilterParameter.PROXY_PORT.getDefaultValue()); + } + } + + public String getPrerenderServiceUrl() { + return prerenderServiceUrl; + } + + public List getBlacklist() { + return blacklist; + } + + public List getWhitelist() { + return whitelist; + } + + public List getExtensions() { + return extensions; + } + + public List getCrawlers() { + return crawlers; + } + + public String getProxy() { + return proxy; + } + + public int getProxyPort() { + return proxyPort; + } +} diff --git a/src/test/java/com/github/greengerong/RenderUtilsTest.java b/src/test/java/com/github/greengerong/RenderUtilsTest.java new file mode 100644 index 0000000..5daaa50 --- /dev/null +++ b/src/test/java/com/github/greengerong/RenderUtilsTest.java @@ -0,0 +1,177 @@ +package com.github.greengerong; + +import com.github.greengerong.config.SeoFilterConfig; +import org.easymock.EasyMock; +import org.easymock.IMocksControl; +import org.junit.Before; +import org.junit.Test; + +import javax.servlet.FilterConfig; +import javax.servlet.http.HttpServletRequest; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static junit.framework.TestCase.assertFalse; +import static org.easymock.EasyMock.expect; +import static org.junit.Assert.assertTrue; + +/** + * Created with IntelliJ IDEA. + * User: set321go + * Date: 12/27/2013 + * Time: 10:16 PM + */ +public class RenderUtilsTest { + HttpServletRequest mockRequest; + SeoFilterConfig mockSeoFilterConfig; + + IMocksControl control; + + @Before + public void initMocks(){ + control = EasyMock.createControl(); + } + + @Test + public void testHasEscapedFragment() throws Exception { + mockRequest = control.createMock(HttpServletRequest.class); + Map paramMap = new HashMap(); + paramMap.put(RenderUtils.ESCAPED_FRAGMENT_PARAMETER, "_escaped_fragment_"); + + expect(mockRequest.getParameterMap()).andReturn(paramMap); + + control.replay(); + assertTrue("The method should have returned true", RenderUtils.hasEscapedFragment(mockRequest)); + control.verify(); + } + + @Test + public void testHasNoEscapedFragment() throws Exception { + mockRequest = control.createMock(HttpServletRequest.class); + Map paramMap = new HashMap(); + paramMap.put("myParameter", "testParam"); + + expect(mockRequest.getParameterMap()).andReturn(paramMap); + + control.replay(); + assertFalse("The method should have returned false", RenderUtils.hasEscapedFragment(mockRequest)); + control.verify(); + } + + @Test + public void testIsValidUserAgent() throws Exception { + mockRequest = control.createMock(HttpServletRequest.class); + mockSeoFilterConfig = control.createMock(SeoFilterConfig.class); + + String testBot = "googlebot"; + List crawlers = new ArrayList(); + crawlers.add(testBot); + + expect(mockRequest.getHeader(RenderUtils.USER_AGENT_HEADER)).andReturn(testBot); + expect(mockSeoFilterConfig.getCrawlers()).andReturn(crawlers); + + control.replay(); + + assertTrue("This method should have returned true", RenderUtils.isValidUserAgent(mockRequest, mockSeoFilterConfig)); + + control.verify(); + } + + @Test + public void testIsNotValidUserAgent() throws Exception { + mockRequest = control.createMock(HttpServletRequest.class); + mockSeoFilterConfig = control.createMock(SeoFilterConfig.class); + + String testBot = "googlebot"; + List crawlers = new ArrayList(); + crawlers.add("testbot"); + + expect(mockRequest.getHeader(RenderUtils.USER_AGENT_HEADER)).andReturn(testBot); + expect(mockSeoFilterConfig.getCrawlers()).andReturn(crawlers); + + control.replay(); + + assertFalse("This method should have returned false", RenderUtils.isValidUserAgent(mockRequest, mockSeoFilterConfig)); + + control.verify(); + } + + @Test + public void testIsInBlackList() throws Exception { + String url = "http://test.com"; + String referer = "http://referer.com"; + List blacklist = new ArrayList(); + blacklist.add("http://test.com"); + + + assertTrue("This method should have returned true", RenderUtils.isInBlackList(url, referer, blacklist)); + } + + @Test + public void testIsNotInBlackList() throws Exception { + String url = "http://test.com"; + String referer = "http://referer.com"; + List blacklist = new ArrayList(); + blacklist.add("http://blacklist.com"); + + + assertFalse("This method should have returned false", RenderUtils.isInBlackList(url, referer, blacklist)); + } + + @Test + public void testIsInWhiteList() throws Exception { + String url = "http://test.com"; + List whitelist = new ArrayList(); + whitelist.add("http://test.com"); + + + assertTrue("This method should have returned true", RenderUtils.isInWhiteList(url, whitelist)); + } + + @Test + public void testIsNotInWhiteList() throws Exception { + String url = "http://test.com"; + List whitelist = new ArrayList(); + whitelist.add("http://dummy.com"); + + + assertFalse("This method should have returned false", RenderUtils.isInWhiteList(url, whitelist)); + } + + @Test + public void testIgnoreFileExtension() throws Exception { + mockSeoFilterConfig = control.createMock(SeoFilterConfig.class); + + String url = "http://test.com/myImg.jpg"; + List extensions = new ArrayList(); + extensions.add(".jpg"); + + expect(mockSeoFilterConfig.getExtensions()).andReturn(extensions); + + control.replay(); + + assertTrue("This method should have returned true", RenderUtils.ignoreFileExtension(url, mockSeoFilterConfig)); + + control.verify(); + } + + @Test + public void testDoNotIgnoreFileExtension() throws Exception { + mockSeoFilterConfig = control.createMock(SeoFilterConfig.class); + + String url = "http://test.com/my-page.html"; + List extensions = new ArrayList(); + extensions.add(".jpg"); + + expect(mockSeoFilterConfig.getExtensions()).andReturn(extensions); + + control.replay(); + + assertFalse("This method should have returned false", RenderUtils.ignoreFileExtension(url, mockSeoFilterConfig)); + + control.verify(); + } +}