From c5fda9faefac4fbdfdb3466e0427f5f9c49a5b19 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 8 Oct 2015 20:20:28 +0800 Subject: [PATCH 01/17] readme & update prerender client --- README.md | 25 +++++++++++-------- demo/pom.xml | 10 ++++---- demo/src/main/webapp/WEB-INF/web.xml | 8 +++--- prerender-java.iml | 3 +-- .../github/greengerong/PrerenderConfig.java | 19 ++++++++------ 5 files changed, 37 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 50d9bb9..1affbe8 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa 2. Check to make sure we aren't requesting a resource (js, css, etc...) 3. (optional) Check to make sure the url is in the whitelist 4. (optional) Check to make sure the url isn't in the blacklist -2. Make a `GET` request to the [prerender service](https://github.com/collectiveip/prerender)(phantomjs server) for the page's prerendered HTML +2. Make a `GET` request to the [prerender service](https://github.com/prerender/prerender)(phantomjs server) for the page's prerendered HTML 3. Return that HTML to the crawler ## Customization @@ -56,7 +56,7 @@ We usually set the original url in an http header which is added by the reverse ### Using your own prerender service -If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://prerender.herokuapp.com` +If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://service.prerender.io/` $ export PRERENDER_SERVICE_URL= @@ -92,16 +92,21 @@ If you want to cache the caching, analytics, log or others, you can config it. I ## Testing -If you want to make sure your pages are rendering correctly: +If your URLs use a hash-bang: -1. Open the Developer Tools in Chrome (Cmd + Atl + J) -2. Click the Settings gear in the bottom right corner. -3. Click "Overrides" on the left side of the settings panel. -4. Check the "User Agent" checkbox. -6. Choose "Other..." from the User Agent dropdown. -7. Type `googlebot` into the input box. -8. Refresh the page (make sure to keep the developer tools open). + If you want to see `http://localhost:3000/#!/profiles/1234` + Then go to `http://localhost:3000/?_escaped_fragment_=/profiles/1234` +If your URLs use push-state: + + If you want to see `http://localhost:3000/profiles/1234` + Then go to `http://localhost:3000/profiles/1234?_escaped_fragment_=` + + +project demo test url: + + http://localhost:8080/test/?_escaped_fragment_= + ## License The MIT License (MIT) diff --git a/demo/pom.xml b/demo/pom.xml index 2552382..076a7bb 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -13,11 +13,11 @@ prerender-java 1.6.2 - - com.github.greengerong - prerender-java-redis - 0.0.1-SNAPSHOT - + + + + + commons-io commons-io diff --git a/demo/src/main/webapp/WEB-INF/web.xml b/demo/src/main/webapp/WEB-INF/web.xml index c5e037b..2677fca 100644 --- a/demo/src/main/webapp/WEB-INF/web.xml +++ b/demo/src/main/webapp/WEB-INF/web.xml @@ -15,10 +15,10 @@ crawlerUserAgents YahooSeeker - - preRenderEventHandler - com.github.greengerong.PrerenderRedisCache - + + + + prerender diff --git a/prerender-java.iml b/prerender-java.iml index cbd3c69..f74aa01 100644 --- a/prerender-java.iml +++ b/prerender-java.iml @@ -23,5 +23,4 @@ - - + \ No newline at end of file diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 1dc1598..90c2240 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -2,7 +2,6 @@ import com.google.common.collect.Lists; -import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; @@ -84,7 +83,8 @@ public String getForwardedURLHeader() { public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview" - , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot"); + , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot", "vkShare", + "W3C_Validator", "redditbot", "Applebot"); final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); if (isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); @@ -94,10 +94,10 @@ public List getCrawlerUserAgents() { } public List getExtensionsToIgnore() { - List extensionsToIgnore = Lists.newArrayList(".xml", ".js", ".css", ".less", ".png", ".jpg", ".jpeg", - ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg", - ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg", - ".iso", ".flv", ".m4v", ".torrent"); + List extensionsToIgnore = Lists.newArrayList(".js", ".json", ".css", ".xml", ".less", ".png", ".jpg", + ".jpeg", ".gif", ".pdf", ".doc", ".txt", ".ico", ".rss", ".zip", ".mp3", ".rar", ".exe", ".wmv", + ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", + ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent", ".woff", ".ttf"); final String extensionsToIgnoreFromConfig = config.get("extensionsToIgnore"); if (isNotBlank(extensionsToIgnoreFromConfig)) { extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(","))); @@ -124,6 +124,11 @@ public List getBlacklist() { public String getPrerenderServiceUrl() { final String prerenderServiceUrl = config.get("prerenderServiceUrl"); - return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : PRERENDER_IO_SERVICE_URL; + return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : getDefaultPrerenderIoServiceUrl(); + } + + private String getDefaultPrerenderIoServiceUrl() { + final String prerenderServiceUrlInEnv = System.getProperty("PRERENDER_SERVICE_URL"); + return isNotBlank(prerenderServiceUrlInEnv) ? prerenderServiceUrlInEnv : PRERENDER_IO_SERVICE_URL; } } From 99cefef83e1b17041dc9f7067c607b9f288fabe8 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 8 Oct 2015 20:27:55 +0800 Subject: [PATCH 02/17] [maven-release-plugin] prepare release prerender-java-1.6.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 112aab4..251e979 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.4-SNAPSHOT + 1.6.4 jar prerender-java From d7b7d943208d973f6dde62be243753a1df4cc99f Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 8 Oct 2015 20:28:19 +0800 Subject: [PATCH 03/17] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 251e979..01017d3 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.github.greengerong prerender-java - 1.6.4 + 1.6.5-SNAPSHOT jar prerender-java From 29a1e08027ae926fe10d3627d027b98c8b717696 Mon Sep 17 00:00:00 2001 From: greengerong Date: Thu, 8 Oct 2015 20:50:54 +0800 Subject: [PATCH 04/17] new version 1.6.4 --- README.md | 2 +- demo/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1affbe8..7d0af15 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa com.github.greengerong prerender-java - 1.6.4-SNAPSHOT + 1.6.4 ## How it works diff --git a/demo/pom.xml b/demo/pom.xml index 076a7bb..5b1c40c 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -11,7 +11,7 @@ com.github.greengerong prerender-java - 1.6.2 + 1.6.4 From 707e199407debb385cd2b0683549ccd099dc5a61 Mon Sep 17 00:00:00 2001 From: Jan Philip Bernius Date: Fri, 6 May 2016 20:03:02 +0200 Subject: [PATCH 05/17] Remove .DS_Store files --- .DS_Store | Bin 6148 -> 0 bytes .gitignore | 3 +++ demo/.DS_Store | Bin 6148 -> 0 bytes demo/src/.DS_Store | Bin 6148 -> 0 bytes demo/src/main/.DS_Store | Bin 6148 -> 0 bytes demo/src/main/webapp/.DS_Store | Bin 6148 -> 0 bytes src/main/.DS_Store | Bin 6148 -> 0 bytes src/main/java/.DS_Store | Bin 6148 -> 0 bytes src/main/java/com/.DS_Store | Bin 6148 -> 0 bytes src/main/java/com/github/.DS_Store | Bin 6148 -> 0 bytes 10 files changed, 3 insertions(+) delete mode 100644 .DS_Store delete mode 100644 demo/.DS_Store delete mode 100644 demo/src/.DS_Store delete mode 100644 demo/src/main/.DS_Store delete mode 100644 demo/src/main/webapp/.DS_Store delete mode 100644 src/main/.DS_Store delete mode 100644 src/main/java/.DS_Store delete mode 100644 src/main/java/com/.DS_Store delete mode 100644 src/main/java/com/github/.DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index f8b64f14cb1b4abba656ea103bb350b269a79956..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKQA+|r5S~@5J1X!c@Nwx`LM>CFheHf{3>19GsJl*5Fi*TQAIv&}eoz0V-_y+Q zDw*{lghb82>^D0*bIW{p?A!u?sE+(RKpFrnRKkRf!xuv9q)U=8o)n^>XL#dq;PnCz z=dVlA?D&lg(7Q{44{f-E;N$!A2EA?=<#KN#kxXrDZdtZ%?X;TRz8{RM!Km+t7xhjs zu6O;m=MFEG9l6mcthz(Ba_&YQTv2u-2zqF@OmwsBN1dy5W-=)j3t5@3RHj)uDV1_r zd30EwPHoGw(=XQE{=vyv!+n@OKE;|lfuEU@U4t`tMq^&aeSO&P1^rv}QG60Y7{bl^ ziz^h1Sln^>CyFygd@5~LJ|i>03@`($F<=f;YkRdcj%Eg!fq%dNoevU~&@mWmR7VF6 z>I#5JzmZDNrn>~?NP~{SSR<~W2%UCFheHf{3>19GsJl*75Kp`_AIv&}eoz0V-_y+Q z8kq$?ghb82>^D0*bIW{p?A!u?s7?F=Kn4ITRKkRf!xuv9q$`pzo)n^>XM`hf5O_Fy z@kT4r?D&lg(7Q`Q2xGYQrXSzmpdUv0{HsVLQ(Nh6%eJlEcB?=1gJ~_84E^x3(F>-H zzTffO@s+Y8H=2YscdS+}+^B~u%5Db10PVJkZuR}Bcb&=3X2nt|CkrQ)c}~vC<$O*a zA64da+p_G;v$cP4czWJ+@8=JXv1U%-N2X-Y-~yh|Fh$%o#=}7{?4pOF(-47+#dV5|kqiItF8nxPl^dDxyvm=87S7I{KxFa}35Bbvg*OGd{=cEX)l> zsNK;oRX7MoBah4gGqB7+Qcp9~|Ifa^|1T%;j2U1C{uBcuRd3X5Sd!_jE5%W-m8dtU pBor5Gd{03`wPK8=R-8puf__N`qGK@Dh#nOF5l}Snzzl4Zfj5k^fwBMq diff --git a/demo/src/.DS_Store b/demo/src/.DS_Store deleted file mode 100644 index e66c6b5437e4c9df30bd8a4979ba83365488deeb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~K}*9h6vvaTYKscJ2s;MPMK*PA2M?u8@F;Y!gA;A3xI%5m+PTB57QxTsxAF7% zUXqH_okipYlK)HIdr9+yEeT_cd-JHnn9Ue7p`vm$))X9G>uTs=RE~3mi+ma;SqS|% z?fdJ3-)^vb7O#ZfW2wp+{PVYlmu&Pl&?#Io0GJL34LUzV0> zTK22Cw|{VYJ`5hqr{_v>)`ghzk? diff --git a/demo/src/main/.DS_Store b/demo/src/main/.DS_Store deleted file mode 100644 index 496035a1af584566e79dc9088a9c3c0ee69cbcd5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z-NTO;8~hp~uB@kyfqk!9yq&JPIlFU_}#BY#=0|NzI{FgW&V{Ha?Ft zyPHEWXAwICv)}Ce>}Eg6{xHV)c$NivHzj+ZCJPu-6qA=cDb` zGz`=FFpf`7uWqL9%l7rHQ2cSATFQ~e65he6Eall*MPVF0fxoONa|ww7Vt^PR26mVM zeJB_QJInzsofsen>KVZOL4YFK8ViMT>wpHY&lqnZqJWKW2}GgM)>tTn2nbiDfU1=1 zCk9vL;1@d2)>tT1<&4XjVH`bk`FP=ScJK?C&bY0RT4I10s4~!0T?^0uYxreqANkc1 z>JbCPz&~SvwyPE(St~#vK~XTTKm!Eq$|V3gxQ}#XP{jq>kY{Tw6yhl8 Rm*s$T5s-vXLk#=^1K(ORNofE8 diff --git a/demo/src/main/webapp/.DS_Store b/demo/src/main/webapp/.DS_Store deleted file mode 100644 index 2a74ced11ee55ebb91e09228cecc47288de92ca0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5T3Qwrl^pM(BtB{N~_lP;30%o^ddqFJy_9%hz*3MG^st*Y7l%L-^S;0 zW_Jr#>s7?g!0tCYKf9R^vOfSI+;LC^umQlrMkq*GB4l3bS~J0fLd|gw7jWuNlA#|h zndmQ?=-Ug>g%-H52cNz__R;g^52G+iE0uS#mdmei6s%&=+Uj+~K@d;ecsvM_t9C!0 zw8P-e_eR%BPrY=UxZX&4F1@sm9;J8UI6}Q=qPt;`_HXRcbb3&$m1Xs~F)PbyyyvNzobEX*-NVbw~gL8O6LHlM(7yKG^(uwJGwrTKTk-4I=v+br9sDFrV%|T zLZ>3?RAHVNLZ_o$nmET`rctMZP%Gm+X65$=PMPZV!OpgLU9=>D5x)70&qb4NJ%-hUy_b- Yj=@YL%|dpSj>s1QMF@Ayz%MZH0p1};X#fBK diff --git a/src/main/.DS_Store b/src/main/.DS_Store deleted file mode 100644 index 1310f12577816836235239bd947be7ffa0766e36..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~K}*9h6vvaTYKscJ2s;MPLpF782M?u8@F;Y!gA;A3xJqfq+HHreS_D6j-^S15 zdr2xzcNUQsNd7N*?V;Vgo?`1*idkIqpPNaQ8~^LEV4-urvdce zwC}GAe!IaQna}3T|N8yGB#yGY)q3Z(y0NvrV_KHEHyTDsm@eFOmW0{0H%=E`6y68^ z^hVmbpU*PapGwb_pO3*K?LnHxV2^b4FbebWt=(8Ik2{^FXrJ|prdW2nt)@6V=@o@# znwI@)9vmK>Uk?0L@$_6OjvO${vahw07cf4zG54lPoF)%2R>fCwq(p!S5CI}U1ggWq z+$9!7fCvzQZv^o2D4?P`T1%z!UI`6epV4oiVgVc95~xC}qqS5D9w=O;f~r)mpBP-F z!?@6Kj@DACN@rZI4D;xftH%qMtHZdE;fy;yPc>Su3F4LB)b`B^pq`Ub_T92mePJ(y9CcZHRNUmP&CJ9GB^! ObPSjXs2s=0rePvOsJ(|% zP`Ha52`+qQPbqd1M^!jtnf$}athT=;Qx7Y4H{ zW#>UY&%9u!JQqPe0gtkWQYK)JP4p;^^2xQ+TrLm0-InN_^oy2Q_ImA>I6mqZg>6~3 z^JeYsADo^KgQw#8rPLHTKvrqjXeF;;m_hZ$W?9eV$IODEXDv1CQSS3)`RR_=iGx%lJKJr()P>l!> zfqzDTHwON|gGHIM^~dt?tQC-NAY;O~5)DXTuUrD4g8N8QIkjJ)4RNl~QY(&v?W#K{ OT?7;mDu}=@5cmYR6Gp25 diff --git a/src/main/java/com/.DS_Store b/src/main/java/com/.DS_Store deleted file mode 100644 index e0512af466f9b9e96547f2cda22eb0628ed9b252..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z-NTO;8~hp~uB@kyfqk!9xfYJPIlFU_}!mO`tTTN$jCUgW&V{Ha?Ft zyPHEWXAwICv)}Ce>}Eg6{xHV4lLdXoEXJ4tipbHZ5p>sv8af$~%P}(M`}Fy>FPNaX6bs%OKu3akdPSJ2#9sZWKKE z-uhnJsh4Jn@n)=NW(NP7~;3$SN8dm064_+WLm+q1!-EBcqCyeqcDVXrGL&PRD} z8ir|o7{@24S2q*yC4YS@6n`A3mU5)Agm*A1OL=zJ%SF7LgTJgPa|ww7Vt^PR26mVM zeJB_QJInzsofsen>KVZOL4YFK8ViMT>wpHY&lqnZqJWKW2}GgM)>tTn2nbiDfU1=1 zCk9vL;1@d2)>tT1<&4XjVH`bk`FP=ScJK?C&bY0RT4I10s4~!0T?^0uYxreqANkc1 z>JbCPz&~Svx5n<+fko-F^~dt?tQDY-peUGEpaBAQ%6b|%Zb3AY~)C*$|BHp}d7^GRV`OZt_%FgbdVVcJNa1ca( zJh$Un7VzV$ta|8{+J=o9Cusn6+2q@aXvBs_#DLPtS#7lEX(Qr5V#Ntt0 b2bBVTfd-(Zu}}ye5c(02G|)f{{3-( Date: Sun, 29 Apr 2018 01:19:14 +0530 Subject: [PATCH 06/17] #44 added provision for specifying protocol --- README.md | 3 +++ .../greengerong/PreRenderSEOFilter.java | 19 +++++++++++-------- .../github/greengerong/PrerenderConfig.java | 4 ++++ .../greengerong/PrerenderSeoService.java | 8 ++++++-- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7d0af15..f2af0fc 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,9 @@ example: someproxy,someproxy1 Important for servers behind reverse proxy that need the public url to be used for pre-rendering. We usually set the original url in an http header which is added by the reverse proxy (similar to the more standard `x-forwarded-proto` and `x-forwarded-for`) +### protocol +If you specifically want to make sure that the Prerender service queries using http or https protocol, you can set the init-param `protocol` to `https` or `http` respectively. + ### Using your own prerender service If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://service.prerender.io/` diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 6eb721f..39195f9 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -1,20 +1,23 @@ package com.github.greengerong; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -import javax.servlet.*; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; public class PreRenderSEOFilter implements Filter { public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl"); + "blacklist", "prerenderServiceUrl", "protocol"); private PrerenderSeoService prerenderSeoService; @Override diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index 90c2240..a18d314 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -79,6 +79,10 @@ public String getPrerenderToken() { public String getForwardedURLHeader() { return config.get("forwardedURLHeader"); } + + public String getProtocol() { + return config.get("protocol"); + } public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("baiduspider", diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 250b075..4ab980e 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -179,12 +179,16 @@ private void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest p } private String getRequestURL(HttpServletRequest request) { - if (prerenderConfig.getForwardedURLHeader() != null) { + if (StringUtils.isNotEmpty(prerenderConfig.getForwardedURLHeader())) { String url = request.getHeader(prerenderConfig.getForwardedURLHeader()); - if (url != null) { + if (StringUtils.isNotEmpty(url)) { return url; } } + if (StringUtils.isNotEmpty(prerenderConfig.getProtocol())) { + String url = request.getRequestURL().toString(); + return url.replace(request.getScheme(), prerenderConfig.getProtocol()); + } return request.getRequestURL().toString(); } From 4f5c76f29e0c1fb63dcd31744bc27da1ca622e7a Mon Sep 17 00:00:00 2001 From: Karan Gujral Date: Sun, 29 Apr 2018 01:23:33 +0530 Subject: [PATCH 07/17] #44 changed readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f2af0fc..0486019 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Important for servers behind reverse proxy that need the public url to be used f We usually set the original url in an http header which is added by the reverse proxy (similar to the more standard `x-forwarded-proto` and `x-forwarded-for`) ### protocol -If you specifically want to make sure that the Prerender service queries using http or https protocol, you can set the init-param `protocol` to `https` or `http` respectively. +If you specifically want to make sure that the Prerender service queries using https or http protocol, you can set the init-param `protocol` to `https` or `http` respectively. ### Using your own prerender service From 049355625b8d074209511987a965538bbd2b082d Mon Sep 17 00:00:00 2001 From: green Date: Mon, 7 May 2018 11:11:19 +0800 Subject: [PATCH 08/17] Update .travis.yml --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index aec209e..e8f05af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: java script: "mvn clean install -U" jdk: - #- oraclejdk6 - - oraclejdk7 - - openjdk6 - - openjdk7 \ No newline at end of file + - openjdk7 + - openjdk8 From 23b6fd6370ced4c38325405c8019e26bf36c6557 Mon Sep 17 00:00:00 2001 From: Karan Gujral Date: Fri, 11 May 2018 14:04:10 -0500 Subject: [PATCH 09/17] trying to re-run CI --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0486019..edf8c2e 100644 --- a/README.md +++ b/README.md @@ -112,4 +112,4 @@ project demo test url: ## License -The MIT License (MIT) +The MIT License (MIT) \ No newline at end of file From b2583beb0a34c0566d7ddf80ce38e767ce7d6776 Mon Sep 17 00:00:00 2001 From: green Date: Wed, 1 Aug 2018 11:38:08 +0800 Subject: [PATCH 10/17] LICENSE --- LICENSE.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..43db8ab --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017-present, GreenGerong(https://github.com/greengerong), Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From d677fb0e3a99f59bd76995e596688d13c1504b3d Mon Sep 17 00:00:00 2001 From: Nathan McWilliams Date: Thu, 23 Aug 2018 17:17:37 -0400 Subject: [PATCH 11/17] change logging of final request url from log.trace to log.info --- src/main/java/com/github/greengerong/PrerenderSeoService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 4ab980e..288e869 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -331,7 +331,7 @@ private boolean beforeRender(HttpServletRequest request, HttpServletResponse res private boolean proxyPrerenderedPageResponse(HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException { final String apiUrl = getApiUrl(getFullUrl(request)); - log.trace(String.format("Prerender proxy will send request to:%s", apiUrl)); + log.info(String.format("Prerender proxy will send request to:%s", apiUrl)); final HttpGet getMethod = getHttpGet(apiUrl); copyRequestHeaders(request, getMethod); withPrerenderToken(getMethod); From f01e29c4c0d590a380143f3b83f75e9a3702478e Mon Sep 17 00:00:00 2001 From: Todd Hooper Date: Mon, 15 Oct 2018 20:51:37 -0500 Subject: [PATCH 12/17] Add a check for our X-Prerender header Make sure requests from the prerender browser don't get sent through to Prerender.io --- .../java/com/github/greengerong/PrerenderSeoService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/github/greengerong/PrerenderSeoService.java b/src/main/java/com/github/greengerong/PrerenderSeoService.java index 4ab980e..043a4a9 100644 --- a/src/main/java/com/github/greengerong/PrerenderSeoService.java +++ b/src/main/java/com/github/greengerong/PrerenderSeoService.java @@ -106,6 +106,11 @@ private boolean shouldShowPrerenderedPage(HttpServletRequest request) throws URI log.trace("request is for a (static) resource; intercept: no"); return false; } + + String prerenderHeader = request.getHeader("X-Prerender"); + if (StringUtils.isNotEmpty(prerenderHeader)) { + return false; + } final List whiteList = prerenderConfig.getWhitelist(); if (whiteList != null && !isInWhiteList(url, whiteList)) { From df2d5438c97ee6f55003ff3337375a27642c5a4a Mon Sep 17 00:00:00 2001 From: Todd Hooper Date: Mon, 15 Oct 2018 20:58:55 -0500 Subject: [PATCH 13/17] added a test for the X-Prerender header check --- .../greengerong/PreRenderSEOFilterTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java index 6776586..ba81d67 100644 --- a/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java +++ b/src/test/java/com/github/greengerong/PreRenderSEOFilterTest.java @@ -132,6 +132,25 @@ public void should_not_handle_when_user_agent_is_not_crawler() throws Exception verify(filterChain).doFilter(servletRequest, servletResponse); } + @Test + public void should_not_handle_when_x_prerender_header_is_present() throws Exception { + //given + when(filterConfig.getInitParameter("crawlerUserAgents")).thenReturn("crawler1,crawler2"); + preRenderSEOFilter.init(filterConfig); + + when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("http://localhost/test")); + when(servletRequest.getMethod()).thenReturn(METHOD_NAME); + when(servletRequest.getParameterMap()).thenReturn(Maps.newHashMap()); + when(servletRequest.getHeader("User-Agent")).thenReturn("crawler1"); + when(servletRequest.getHeader("X-Prerender")).thenReturn("1"); + //when + preRenderSEOFilter.doFilter(servletRequest, servletResponse, filterChain); + + //then + verify(httpClient, never()).execute(httpGet); + verify(filterChain).doFilter(servletRequest, servletResponse); + } + @Test public void should_not_handle_when_url_is_a_resource() throws Exception { //given From 09a90437c5e848956842a449698cda13254c5c45 Mon Sep 17 00:00:00 2001 From: Csaba Varro Date: Wed, 24 Nov 2021 13:58:45 +0100 Subject: [PATCH 14/17] Extend user agent list --- src/main/java/com/github/greengerong/PrerenderConfig.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/greengerong/PrerenderConfig.java b/src/main/java/com/github/greengerong/PrerenderConfig.java index a18d314..6b4e894 100644 --- a/src/main/java/com/github/greengerong/PrerenderConfig.java +++ b/src/main/java/com/github/greengerong/PrerenderConfig.java @@ -79,7 +79,7 @@ public String getPrerenderToken() { public String getForwardedURLHeader() { return config.get("forwardedURLHeader"); } - + public String getProtocol() { return config.get("protocol"); } @@ -88,7 +88,9 @@ public List getCrawlerUserAgents() { List crawlerUserAgents = Lists.newArrayList("baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview" , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot", "vkShare", - "W3C_Validator", "redditbot", "Applebot"); + "W3C_Validator", "redditbot", "Applebot", "whatsapp", "flipboard", "tumblr", "bitlybot", + "skypeuripreview", "nuzzel", "discordbot", "google page speed", "qwantify", "pinterestbot", + "bitrix link preview", "xing-contenttabreceiver", "chrome-lighthouse", "telegrambot"); final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents"); if (isNotBlank(crawlerUserAgentsFromConfig)) { crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(","))); From 6c415e843ce143e2eab3a21b27b00b143d68ba0b Mon Sep 17 00:00:00 2001 From: greengerong Date: Mon, 12 Jun 2023 00:38:18 +0800 Subject: [PATCH 15/17] rename word --- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 39195f9..1fa4719 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -28,9 +28,9 @@ public void init(FilterConfig filterConfig) throws ServletException { @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - boolean isPrerendered = prerenderSeoService.prerenderIfEligible( + boolean isPreRendered = prerenderSeoService.prerenderIfEligible( (HttpServletRequest) servletRequest, (HttpServletResponse) servletResponse); - if (!isPrerendered) { + if (!isPreRendered) { filterChain.doFilter(servletRequest, servletResponse); } } From 17618ffac877c9e0aa755f18c6bfb1ac103da8a3 Mon Sep 17 00:00:00 2001 From: green Date: Sat, 17 Jun 2023 11:41:14 +0800 Subject: [PATCH 16/17] Update PreRenderSEOFilter.java socketTimeout --- src/main/java/com/github/greengerong/PreRenderSEOFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java index 1fa4719..5704025 100644 --- a/src/main/java/com/github/greengerong/PreRenderSEOFilter.java +++ b/src/main/java/com/github/greengerong/PreRenderSEOFilter.java @@ -17,7 +17,7 @@ public class PreRenderSEOFilter implements Filter { public static final List PARAMETER_NAMES = Lists.newArrayList("preRenderEventHandler", "proxy", "proxyPort", "prerenderToken", "forwardedURLHeader", "crawlerUserAgents", "extensionsToIgnore", "whitelist", - "blacklist", "prerenderServiceUrl", "protocol"); + "blacklist", "prerenderServiceUrl", "protocol","socketTimeout"); private PrerenderSeoService prerenderSeoService; @Override From 292bd36335126dd9ea3fd8329ce629efddeeb91c Mon Sep 17 00:00:00 2001 From: green Date: Sun, 9 Jul 2023 16:28:03 +0800 Subject: [PATCH 17/17] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index edf8c2e..4d3f00f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Prerender Java [![Build Status](https://travis-ci.org/greengerong/prerender-java.png)](https://travis-ci.org/greengerong/prerender-java) +Prerender Java =========================== Are you using backbone, angular, emberjs, etc, but you're unsure about the SEO implications? @@ -112,4 +112,4 @@ project demo test url: ## License -The MIT License (MIT) \ No newline at end of file +The MIT License (MIT)