diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index f8b64f1..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore index 71d7773..112f2f2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ target/ **/target/ **/.idea/ *.iml + +### OSX ### +.DS_Store diff --git a/.travis.yml b/.travis.yml index aec209e..e8f05af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: java script: "mvn clean install -U" jdk: - #- oraclejdk6 - - oraclejdk7 - - openjdk6 - - openjdk7 \ No newline at end of file + - openjdk7 + - openjdk8 diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..43db8ab --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017-present, GreenGerong(https://github.com/greengerong), Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 50d9bb9..4d3f00f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Prerender Java [![Build Status](https://travis-ci.org/greengerong/prerender-java.png)](https://travis-ci.org/greengerong/prerender-java) +Prerender Java =========================== Are you using backbone, angular, emberjs, etc, but you're unsure about the SEO implications? @@ -29,7 +29,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.6.4-SNAPSHOT + 1.6.4 ## How it works @@ -38,7 +38,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa 2. Check to make sure we aren't requesting a resource (js, css, etc...) 3. (optional) Check to make sure the url is in the whitelist 4. (optional) Check to make sure the url isn't in the blacklist -2. Make a `GET` request to the [prerender service](https://github.com/collectiveip/prerender)(phantomjs server) for the page's prerendered HTML +2. Make a `GET` request to the [prerender service](https://github.com/prerender/prerender)(phantomjs server) for the page's prerendered HTML 3. Return that HTML to the crawler ## Customization @@ -54,9 +54,12 @@ example: someproxy,someproxy1 Important for servers behind reverse proxy that need the public url to be used for pre-rendering. We usually set the original url in an http header which is added by the reverse proxy (similar to the more standard `x-forwarded-proto` and `x-forwarded-for`) +### protocol +If you specifically want to make sure that the Prerender service queries using https or http protocol, you can set the init-param `protocol` to `https` or `http` respectively. + ### Using your own prerender service -If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://prerender.herokuapp.com` +If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://service.prerender.io/` $ export PRERENDER_SERVICE_URL= @@ -92,16 +95,21 @@ If you want to cache the caching, analytics, log or others, you can config it. I ## Testing -If you want to make sure your pages are rendering correctly: +If your URLs use a hash-bang: + + If you want to see `http://localhost:3000/#!/profiles/1234` + Then go to `http://localhost:3000/?_escaped_fragment_=/profiles/1234` + +If your URLs use push-state: + + If you want to see `http://localhost:3000/profiles/1234` + Then go to `http://localhost:3000/profiles/1234?_escaped_fragment_=` + -1. Open the Developer Tools in Chrome (Cmd + Atl + J) -2. Click the Settings gear in the bottom right corner. -3. Click "Overrides" on the left side of the settings panel. -4. Check the "User Agent" checkbox. -6. Choose "Other..." from the User Agent dropdown. -7. Type `googlebot` into the input box. -8. Refresh the page (make sure to keep the developer tools open). +project demo test url: + http://localhost:8080/test/?_escaped_fragment_= + ## License The MIT License (MIT) diff --git a/demo/.DS_Store b/demo/.DS_Store deleted file mode 100644 index 8b95c52..0000000 Binary files a/demo/.DS_Store and /dev/null differ diff --git a/demo/pom.xml b/demo/pom.xml index 2552382..5b1c40c 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,13 +11,13 @@ com.github.greengerong prerender-java - 1.6.2 - - - com.github.greengerong - prerender-java-redis - 0.0.1-SNAPSHOT + 1.6.4 + + + + + commons-io commons-io diff --git a/demo/src/.DS_Store b/demo/src/.DS_Store deleted file mode 100644 index e66c6b5..0000000 Binary files a/demo/src/.DS_Store and /dev/null differ diff --git a/demo/src/main/.DS_Store b/demo/src/main/.DS_Store deleted file mode 100644 index 496035a..0000000 Binary files a/demo/src/main/.DS_Store and /dev/null differ diff --git a/demo/src/main/webapp/.DS_Store b/demo/src/main/webapp/.DS_Store deleted file mode 100644 index 2a74ced..0000000 Binary files a/demo/src/main/webapp/.DS_Store and /dev/null differ diff --git a/demo/src/main/webapp/WEB-INF/web.xml b/demo/src/main/webapp/WEB-INF/web.xml index c5e037b..2677fca 100644 --- a/demo/src/main/webapp/WEB-INF/web.xml +++ b/demo/src/main/webapp/WEB-INF/web.xml @@ -15,10 +15,10 @@ crawlerUserAgents YahooSeeker - - preRenderEventHandler - com.github.greengerong.PrerenderRedisCache - + + + + prerender diff --git a/pom.xml b/pom.xml index 112aab4..01017d3 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.4-SNAPSHOT + 1.6.5-SNAPSHOT jar prerender-java diff --git a/prerender-java.iml b/prerender-java.iml index cbd3c69..f74aa01 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -23,5 +23,4 @@ - - + \ No newline at end of file diff --git a/src/main/.DS_Store b/src/main/.DS_Store deleted file mode 100644 index 1310f12..0000000 Binary files a/src/main/.DS_Store and /dev/null differ diff --git a/src/main/java/.DS_Store b/src/main/java/.DS_Store deleted file mode 100644 index 666f71f..0000000 Binary files a/src/main/java/.DS_Store and /dev/null differ diff --git a/src/main/java/com/.DS_Store b/src/main/java/com/.DS_Store deleted file mode 100644 index e0512af..0000000 Binary files a/src/main/java/com/.DS_Store and /dev/null differ diff --git a/src/main/java/com/github/.DS_Store b/src/main/java/com/github/.DS_Store deleted file mode 100644 index 162ded6..0000000 Binary files a/src/main/java/com/github/.DS_Store and /dev/null differ diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 6eb721f..5704025 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,20 +1,23 @@ package com.github.greengerong; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -import javax.servlet.*; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; public class PreRenderSEOFilter implements Filter { public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl"); + "blacklist", "prerenderServiceUrl", "protocol","socketTimeout"); private PrerenderSeoService prerenderSeoService; @Override @@ -25,9 +28,9 @@ public void init(FilterConfig filterConfig) throws ServletException { @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - boolean isPrerendered = prerenderSeoService.prerenderIfEligible( + boolean isPreRendered = prerenderSeoService.prerenderIfEligible( (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); - if (!isPrerendered) { + if (!isPreRendered) { filterChain.doFilter(servletRequest, servletResponse); } } diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 1dc1598..6b4e894 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -2,7 +2,6 @@ import com.google.common.collect.Lists; -import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; @@ -81,10 +80,17 @@ public String getForwardedURLHeader() { return config.get("forwardedURLHeader"); } + public String getProtocol() { + return config.get("protocol"); + } + public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview" - , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot"); + , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot", "vkShare", + "W3C_Validator", "redditbot", "Applebot", "whatsapp", "flipboard", "tumblr", "bitlybot", + "skypeuripreview", "nuzzel", "discordbot", "google page speed", "qwantify", "pinterestbot", + "bitrix link preview", "xing-contenttabreceiver", "chrome-lighthouse", "telegrambot"); final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); if (isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); @@ -94,10 +100,10 @@ public List getCrawlerUserAgents() { } public List getExtensionsToIgnore() { - List extensionsToIgnore = Lists.newArrayList(".xml", ".js", ".css", ".less", ".png", ".jpg", ".jpeg", - ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", - ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", - ".iso", ".flv", ".m4v", ".torrent"); + List extensionsToIgnore = Lists.newArrayList(".js", ".json", ".css", ".xml", ".less", ".png", ".jpg", + ".jpeg", ".gif", ".pdf", ".doc", ".txt", ".ico", ".rss", ".zip", ".mp3", ".rar", ".exe", ".wmv", + ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", + ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent", ".woff", ".ttf"); final String extensionsToIgnoreFromConfig = config.get("extensionsToIgnore"); if (isNotBlank(extensionsToIgnoreFromConfig)) { extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); @@ -124,6 +130,11 @@ public List getBlacklist() { public String getPrerenderServiceUrl() { final String prerenderServiceUrl = config.get("prerenderServiceUrl"); - return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : PRERENDER_IO_SERVICE_URL; + return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : getDefaultPrerenderIoServiceUrl(); + } + + private String getDefaultPrerenderIoServiceUrl() { + final String prerenderServiceUrlInEnv = System.getProperty("PRERENDER_SERVICE_URL"); + return isNotBlank(prerenderServiceUrlInEnv) ? prerenderServiceUrlInEnv : PRERENDER_IO_SERVICE_URL; } } diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 250b075..63d3352 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -106,6 +106,11 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI log.trace("request is for a (static) resource; intercept: no"); return false; } + + String prerenderHeader = request.getHeader("X-Prerender"); + if (StringUtils.isNotEmpty(prerenderHeader)) { + return false; + } final List whiteList = prerenderConfig.getWhitelist(); if (whiteList != null && !isInWhiteList(url, whiteList)) { @@ -179,12 +184,16 @@ private void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest p } private String getRequestURL(HttpServletRequest request) { - if (prerenderConfig.getForwardedURLHeader() != null) { + if (StringUtils.isNotEmpty(prerenderConfig.getForwardedURLHeader())) { String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); - if (url != null) { + if (StringUtils.isNotEmpty(url)) { return url; } } + if (StringUtils.isNotEmpty(prerenderConfig.getProtocol())) { + String url = request.getRequestURL().toString(); + return url.replace(request.getScheme(), prerenderConfig.getProtocol()); + } return request.getRequestURL().toString(); } @@ -327,7 +336,7 @@ private boolean beforeRender(HttpServletRequest request, HttpServletResponse res private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException { final String apiUrl = getApiUrl(getFullUrl(request)); - log.trace(String.format("Prerender proxy will send request to:%s", apiUrl)); + log.info(String.format("Prerender proxy will send request to:%s", apiUrl)); final HttpGet getMethod = getHttpGet(apiUrl); copyRequestHeaders(request, getMethod); withPrerenderToken(getMethod); diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 6776586..ba81d67 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -132,6 +132,25 @@ public void should_not_handle_when_user_agent_is_not_crawler() throws Exception verify(filterChain).doFilter(servletRequest, servletResponse); } + @Test + public void should_not_handle_when_x_prerender_header_is_present() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + when(servletRequest.getHeader("X-Prerender")).thenReturn("1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + @Test public void should_not_handle_when_url_is_a_resource() throws Exception { //given