From f1297bbca6003a4e4ca73ef89636188569fbbc98 Mon Sep 17 00:00:00 2001 From: Karan Gujral Date: Sun, 29 Apr 2018 01:19:14 +0530 Subject: [PATCH 01/12] #44 added provision for specifying protocol --- README.md | 3 +++ .../greengerong/PreRenderSEOFilter.java | 19 +++++++++++-------- .../github/greengerong/PrerenderConfig.java | 4 ++++ .../greengerong/PrerenderSeoService.java | 8 ++++++-- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7d0af15..f2af0fc 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,9 @@ example: someproxy,someproxy1 Important for servers behind reverse proxy that need the public url to be used for pre-rendering. We usually set the original url in an http header which is added by the reverse proxy (similar to the more standard `x-forwarded-proto` and `x-forwarded-for`) +### protocol +If you specifically want to make sure that the Prerender service queries using http or https protocol, you can set the init-param `protocol` to `https` or `http` respectively. + ### Using your own prerender service If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://service.prerender.io/` diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 6eb721f..39195f9 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,20 +1,23 @@ package com.github.greengerong; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -import javax.servlet.*; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; public class PreRenderSEOFilter implements Filter { public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl"); + "blacklist", "prerenderServiceUrl", "protocol"); private PrerenderSeoService prerenderSeoService; @Override diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 90c2240..a18d314 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -79,6 +79,10 @@ public String getPrerenderToken() { public String getForwardedURLHeader() { return config.get("forwardedURLHeader"); } + + public String getProtocol() { + return config.get("protocol"); + } public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("baiduspider", diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 250b075..4ab980e 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -179,12 +179,16 @@ private void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest p } private String getRequestURL(HttpServletRequest request) { - if (prerenderConfig.getForwardedURLHeader() != null) { + if (StringUtils.isNotEmpty(prerenderConfig.getForwardedURLHeader())) { String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); - if (url != null) { + if (StringUtils.isNotEmpty(url)) { return url; } } + if (StringUtils.isNotEmpty(prerenderConfig.getProtocol())) { + String url = request.getRequestURL().toString(); + return url.replace(request.getScheme(), prerenderConfig.getProtocol()); + } return request.getRequestURL().toString(); } From 4f5c76f29e0c1fb63dcd31744bc27da1ca622e7a Mon Sep 17 00:00:00 2001 From: Karan Gujral Date: Sun, 29 Apr 2018 01:23:33 +0530 Subject: [PATCH 02/12] #44 changed readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f2af0fc..0486019 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Important for servers behind reverse proxy that need the public url to be used f We usually set the original url in an http header which is added by the reverse proxy (similar to the more standard `x-forwarded-proto` and `x-forwarded-for`) ### protocol -If you specifically want to make sure that the Prerender service queries using http or https protocol, you can set the init-param `protocol` to `https` or `http` respectively. +If you specifically want to make sure that the Prerender service queries using https or http protocol, you can set the init-param `protocol` to `https` or `http` respectively. ### Using your own prerender service From 049355625b8d074209511987a965538bbd2b082d Mon Sep 17 00:00:00 2001 From: green Date: Mon, 7 May 2018 11:11:19 +0800 Subject: [PATCH 03/12] Update .travis.yml --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index aec209e..e8f05af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: java script: "mvn clean install -U" jdk: - #- oraclejdk6 - - oraclejdk7 - - openjdk6 - - openjdk7 \ No newline at end of file + - openjdk7 + - openjdk8 From 23b6fd6370ced4c38325405c8019e26bf36c6557 Mon Sep 17 00:00:00 2001 From: Karan Gujral Date: Fri, 11 May 2018 14:04:10 -0500 Subject: [PATCH 04/12] trying to re-run CI --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0486019..edf8c2e 100644 --- a/README.md +++ b/README.md @@ -112,4 +112,4 @@ project demo test url: ## License -The MIT License (MIT) +The MIT License (MIT) \ No newline at end of file From b2583beb0a34c0566d7ddf80ce38e767ce7d6776 Mon Sep 17 00:00:00 2001 From: green Date: Wed, 1 Aug 2018 11:38:08 +0800 Subject: [PATCH 05/12] LICENSE --- LICENSE.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..43db8ab --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017-present, GreenGerong(https://github.com/greengerong), Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From d677fb0e3a99f59bd76995e596688d13c1504b3d Mon Sep 17 00:00:00 2001 From: Nathan McWilliams Date: Thu, 23 Aug 2018 17:17:37 -0400 Subject: [PATCH 06/12] change logging of final request url from log.trace to log.info --- src/main/java/com/github/greengerong/PrerenderSeoService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 4ab980e..288e869 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -331,7 +331,7 @@ private boolean beforeRender(HttpServletRequest request, HttpServletResponse res private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException { final String apiUrl = getApiUrl(getFullUrl(request)); - log.trace(String.format("Prerender proxy will send request to:%s", apiUrl)); + log.info(String.format("Prerender proxy will send request to:%s", apiUrl)); final HttpGet getMethod = getHttpGet(apiUrl); copyRequestHeaders(request, getMethod); withPrerenderToken(getMethod); From f01e29c4c0d590a380143f3b83f75e9a3702478e Mon Sep 17 00:00:00 2001 From: Todd Hooper Date: Mon, 15 Oct 2018 20:51:37 -0500 Subject: [PATCH 07/12] Add a check for our X-Prerender header Make sure requests from the prerender browser don't get sent through to Prerender.io --- .../java/com/github/greengerong/PrerenderSeoService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 4ab980e..043a4a9 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -106,6 +106,11 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI log.trace("request is for a (static) resource; intercept: no"); return false; } + + String prerenderHeader = request.getHeader("X-Prerender"); + if (StringUtils.isNotEmpty(prerenderHeader)) { + return false; + } final List whiteList = prerenderConfig.getWhitelist(); if (whiteList != null && !isInWhiteList(url, whiteList)) { From df2d5438c97ee6f55003ff3337375a27642c5a4a Mon Sep 17 00:00:00 2001 From: Todd Hooper Date: Mon, 15 Oct 2018 20:58:55 -0500 Subject: [PATCH 08/12] added a test for the X-Prerender header check --- .../greengerong/PreRenderSEOFilterTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 6776586..ba81d67 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -132,6 +132,25 @@ public void should_not_handle_when_user_agent_is_not_crawler() throws Exception verify(filterChain).doFilter(servletRequest, servletResponse); } + @Test + public void should_not_handle_when_x_prerender_header_is_present() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + when(servletRequest.getHeader("X-Prerender")).thenReturn("1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + @Test public void should_not_handle_when_url_is_a_resource() throws Exception { //given From 09a90437c5e848956842a449698cda13254c5c45 Mon Sep 17 00:00:00 2001 From: Csaba Varro Date: Wed, 24 Nov 2021 13:58:45 +0100 Subject: [PATCH 09/12] Extend user agent list --- src/main/java/com/github/greengerong/PrerenderConfig.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index a18d314..6b4e894 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -79,7 +79,7 @@ public String getPrerenderToken() { public String getForwardedURLHeader() { return config.get("forwardedURLHeader"); } - + public String getProtocol() { return config.get("protocol"); } @@ -88,7 +88,9 @@ public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview" , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot", "vkShare", - "W3C_Validator", "redditbot", "Applebot"); + "W3C_Validator", "redditbot", "Applebot", "whatsapp", "flipboard", "tumblr", "bitlybot", + "skypeuripreview", "nuzzel", "discordbot", "google page speed", "qwantify", "pinterestbot", + "bitrix link preview", "xing-contenttabreceiver", "chrome-lighthouse", "telegrambot"); final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); if (isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); From 6c415e843ce143e2eab3a21b27b00b143d68ba0b Mon Sep 17 00:00:00 2001 From: greengerong Date: Mon, 12 Jun 2023 00:38:18 +0800 Subject: [PATCH 10/12] rename word --- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 39195f9..1fa4719 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -28,9 +28,9 @@ public void init(FilterConfig filterConfig) throws ServletException { @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - boolean isPrerendered = prerenderSeoService.prerenderIfEligible( + boolean isPreRendered = prerenderSeoService.prerenderIfEligible( (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); - if (!isPrerendered) { + if (!isPreRendered) { filterChain.doFilter(servletRequest, servletResponse); } } From 17618ffac877c9e0aa755f18c6bfb1ac103da8a3 Mon Sep 17 00:00:00 2001 From: green Date: Sat, 17 Jun 2023 11:41:14 +0800 Subject: [PATCH 11/12] Update PreRenderSEOFilter.java socketTimeout --- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 1fa4719..5704025 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -17,7 +17,7 @@ public class PreRenderSEOFilter implements Filter { public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl", "protocol"); + "blacklist", "prerenderServiceUrl", "protocol","socketTimeout"); private PrerenderSeoService prerenderSeoService; @Override From 292bd36335126dd9ea3fd8329ce629efddeeb91c Mon Sep 17 00:00:00 2001 From: green Date: Sun, 9 Jul 2023 16:28:03 +0800 Subject: [PATCH 12/12] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index edf8c2e..4d3f00f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Prerender Java [![Build Status](https://travis-ci.org/greengerong/prerender-java.png)](https://travis-ci.org/greengerong/prerender-java) +Prerender Java =========================== Are you using backbone, angular, emberjs, etc, but you're unsure about the SEO implications? @@ -112,4 +112,4 @@ project demo test url: ## License -The MIT License (MIT) \ No newline at end of file +The MIT License (MIT)