From 32337a7def038cb8fed16a3def786b59c1668aef Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Mon, 4 Apr 2016 16:47:17 +1000 Subject: [PATCH 01/23] Added HTTPS support A root SSL certificate is generated when the server starts up; this is used to man-in-the-middle HTTPS requests. The client will still be able to tell that their connection is compromised, since by default they won't trust the new CA. --- Dockerfile | 45 ++++++++++++++++++++++++++++++++++++----- detect-apt-proxy.sh | 20 ++++++++++++++++++ run.sh | 49 +++++++++++++++++++-------------------------- squid.conf | 9 +++++++-- squid3.patch | 13 ++++++++++++ start_squid.sh | 42 +++++++++++++++++++++++++++++--------- 6 files changed, 133 insertions(+), 45 deletions(-) create mode 100755 detect-apt-proxy.sh create mode 100644 squid3.patch diff --git a/Dockerfile b/Dockerfile index c6f42c8..685e77d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,46 @@ -FROM silarsis/base -MAINTAINER Kevin Littlejohn -RUN apt-get -yq update +FROM ubuntu:14.04 + +MAINTAINER Alex Fraser + +# Run a caching proxy on the host and bind a port to APT_PROXY_PORT to cache +# apt requests. Build with `docker build --build-arg APT_PROXY_PORT=[X] [...]`. +WORKDIR /root +ARG APT_PROXY_PORT= +COPY detect-apt-proxy.sh /root/ +RUN export DEBIAN_FRONTEND=noninteractive TERM=linux \ + && ./detect-apt-proxy.sh ${APT_PROXY_PORT} no \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + dpkg-dev \ + iptables \ + libssl-dev \ + patch \ + squid-langpack \ + ssl-cert \ + && apt-get source -y squid3 squid-langpack \ + && apt-get build-dep -y squid3 squid-langpack +# rm -rf /var/lib/apt/lists/* \ +# /etc/apt/apt.conf.d/30proxy \ + +# It's silly, but run dpkg-buildpackage again if it fails the first time. This +# is needed because sometimes the `configure` script is busy when building in +# Docker after autoconf sets its mode +x. +COPY squid3.patch /root/ +RUN cd squid3-3.?.? \ + && patch -p1 < /root/squid3.patch \ + && export NUM_PROCS=`grep -c ^processor /proc/cpuinfo` \ + && (dpkg-buildpackage -b -j${NUM_PROCS} || dpkg-buildpackage -b -j${NUM_PROCS}) +RUN dpkg -i \ + squid3-common_3.?.?-?ubuntu?.?_all.deb \ + squid3_3.?.?-?ubuntu?.?_*.deb \ + && mkdir -p /etc/squid3/ssl_cert -RUN apt-get -yq install squid iptables ADD squid.conf /etc/squid3/squid.conf ADD start_squid.sh /usr/local/bin/start_squid.sh -EXPOSE 3128 +VOLUME /var/spool/squid3 +EXPOSE 3128 3129 CMD ["/usr/local/bin/start_squid.sh"] diff --git a/detect-apt-proxy.sh b/detect-apt-proxy.sh new file mode 100755 index 0000000..8db0031 --- /dev/null +++ b/detect-apt-proxy.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# If the host is running a web proxy, use it for apt. +# Adapted from https://gist.github.com/dergachev/8441335 + +APT_PROXY_PORT=$1 +PROXY_LAUNCHPAD=${2:-no} + +HOST_IP=$(route -n | awk '/^0.0.0.0/ {print $2}') +nc -z "$HOST_IP" ${APT_PROXY_PORT} + +if [ $? -eq 0 ]; then + echo "Acquire::http::Proxy \"http://$HOST_IP:$APT_PROXY_PORT\";" >> /etc/apt/apt.conf.d/30proxy + if [ "$PROXY_LAUNCHPAD" = yes ]; then + echo "Acquire::http::Proxy::ppa.launchpad.net DIRECT;" >> /etc/apt/apt.conf.d/30proxy + fi + echo "Using host's apt proxy" +else + echo "No apt proxy detected on Docker host" +fi diff --git a/run.sh b/run.sh index 2bf43fa..b6fb3d7 100755 --- a/run.sh +++ b/run.sh @@ -3,18 +3,13 @@ # Script to maintain ip rules on the host when starting up a transparent # proxy server for docker. -CACHEDIR="/tmp/squid3" # Change this to place the cache somewhere else +CACHEDIR=${CACHEDIR:-/tmp/squid3} +CONTAINER_NAME=${CONTAINER_NAME:-docker-proxy} set -e -# Guard for my own scripts -# Note, if you're running this script direct, it will rebuild if it can't see -# the image. -[ -z ${RUNNING_DRUN} ] && { - RUN_DOCKER="docker run" - CONTAINER_NAME='docker-proxy' - docker images | grep "^${CONTAINER_NAME} " >/dev/null || docker build -q --rm -t ${CONTAINER_NAME} "$(dirname $0)" -} +sudo docker images | grep -q "^${CONTAINER_NAME} " \ + || (echo "Build ${CONTAINER_NAME} image first" && exit 1) start_routing () { # Add a new route table that routes everything marked through the new container @@ -28,13 +23,15 @@ start_routing () { sudo ln -s /usr/local/etc/iproute2/rt_tables /etc/iproute2/rt_tables fi fi - ([ -e /etc/iproute2/rt_tables ] && grep TRANSPROXY /etc/iproute2/rt_tables >/dev/null) || \ - sudo sh -c "echo '1 TRANSPROXY' >> /etc/iproute2/rt_tables" - ip rule show | grep TRANSPROXY >/dev/null || \ - sudo ip rule add from all fwmark 0x1 lookup TRANSPROXY + ([ -e /etc/iproute2/rt_tables ] && grep -q TRANSPROXY /etc/iproute2/rt_tables) \ + || sudo sh -c "echo '1 TRANSPROXY' >> /etc/iproute2/rt_tables" + ip rule show | grep -q TRANSPROXY \ + || sudo ip rule add from all fwmark 0x1 lookup TRANSPROXY sudo ip route add default via "${IPADDR}" dev docker0 table TRANSPROXY - # Mark packets to port 80 external, so they route through the new route table + # Mark packets to port 80 and 443 external, so they route through the new + # route table sudo iptables -t mangle -I PREROUTING -p tcp --dport 80 \! -s "${IPADDR}" -i docker0 -j MARK --set-mark 1 + sudo iptables -t mangle -I PREROUTING -p tcp --dport 443 \! -s "${IPADDR}" -i docker0 -j MARK --set-mark 1 # Exemption rule to stop docker from masquerading traffic routed to the # transparent proxy sudo iptables -t nat -I POSTROUTING -o docker0 -s 172.17.0.0/16 -j ACCEPT @@ -44,22 +41,19 @@ stop_routing () { # Remove the appropriate rules - that is, those that mention the IP Address. set +e [ "x$IPADDR" != "x" ] && { - ip route show table TRANSPROXY | grep default >/dev/null && \ - sudo ip route del default table TRANSPROXY - sudo iptables -t mangle -L PREROUTING -n | grep 'tcp dpt:80 MARK set 0x1' >/dev/null && \ - sudo iptables -t mangle -D PREROUTING -p tcp --dport 80 \! -s "${IPADDR}" -i docker0 -j MARK --set-mark 1 + ip route show table TRANSPROXY | grep -q default \ + && sudo ip route del default table TRANSPROXY + sudo iptables -t mangle -L PREROUTING -n | grep -q 'tcp dpt:80 MARK set 0x1' \ + && sudo iptables -t mangle -D PREROUTING -p tcp --dport 80 \! -s "${IPADDR}" -i docker0 -j MARK --set-mark 1 \ + && sudo iptables -t mangle -D PREROUTING -p tcp --dport 443 \! -s "${IPADDR}" -i docker0 -j MARK --set-mark 1 \ sudo iptables -t nat -D POSTROUTING -o docker0 -s 172.17.0.0/16 -j ACCEPT 2>/dev/null } set -e } stop () { - # Ideally we'd leave the container around and re-use it, but I really - # need a nice way to query for a named container first. Doesn't cost much - # to create a new container anyway, especially given the cache volume is mapped. set +e - docker kill ${CONTAINER_NAME} >/dev/null 2>&1 - docker rm ${CONTAINER_NAME} >/dev/null 2>&1 + sudo docker rm -f ${CONTAINER_NAME} >/dev/null 2>&1 set -e stop_routing } @@ -85,18 +79,17 @@ run () { # Because we're named, make sure the container doesn't already exist stop # Run and find the IP for the running container - CID=$(${RUN_DOCKER} --privileged -d -v "${CACHEDIR}":/var/spool/squid3 --name ${CONTAINER_NAME} ${CONTAINER_NAME}) - IPADDR=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${CID}) + CID=$(sudo docker run --privileged -d -v "${CACHEDIR}":/var/spool/squid3 --name ${CONTAINER_NAME} ${CONTAINER_NAME}) + IPADDR=$(sudo docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${CID}) start_routing # Run at console, kill cleanly if ctrl-c is hit trap interrupted INT trap terminated TERM echo 'Now entering wait, please hit "ctrl-c" to kill proxy and undo routing' - docker logs -f "${CID}" + sudo docker logs -f "${CID}" echo 'Squid exited unexpectedly, cleaning up...' stop } -# Guard so I can include this script into my own scripts -[ -z ${RUNNING_DRUN} ] && run +run echo diff --git a/squid.conf b/squid.conf index 23fc6ab..d53dcf4 100644 --- a/squid.conf +++ b/squid.conf @@ -10,6 +10,7 @@ acl Safe_ports port 488 # gss-http acl Safe_ports port 591 # filemaker acl Safe_ports port 777 # multiling http acl CONNECT method CONNECT + http_access deny !Safe_ports http_access deny CONNECT !SSL_ports http_access allow localhost manager @@ -17,15 +18,19 @@ http_access deny manager http_access allow localhost http_access allow all #http_access deny all -http_port 3128 transparent + +http_port 3128 intercept +https_port 3129 cert=/etc/squid3/ssl_cert/myCA.pem generate-host-certificates=on intercept ssl-bump + maximum_object_size 512 MB cache_dir ufs /var/spool/squid3 2024 2 8 cache_mem 200 MB maximum_object_size_in_memory 100 MB cache_replacement_policy heap LFUDA + refresh_pattern Packages\.bz2$ 0 20% 4320 refresh-ims refresh_pattern Sources\.bz2$ 0 20% 4320 refresh-ims refresh_pattern Release\.gpg$ 0 20% 4320 refresh-ims refresh_pattern Release$ 0 20% 4320 refresh-ims -refresh_pattern -i .(udeb|tar.gz|deb|rpm|exe|zip|tar|tgz|bz2|ram|rar|bin)$ 129600 100% 129600 override-expire ignore-no-cache ignore-no-store +refresh_pattern -i .(udeb|tar.gz|deb|rpm|exe|zip|tar|tgz|bz2|ram|rar|bin|jar)$ 129600 100% 129600 override-expire ignore-no-cache ignore-no-store refresh_pattern . 0 20% 4320 diff --git a/squid3.patch b/squid3.patch new file mode 100644 index 0000000..49a903a --- /dev/null +++ b/squid3.patch @@ -0,0 +1,13 @@ +--- a/debian/rules 2014-02-18 03:13:28.000000000 +0000 ++++ b/debian/rules 2016-04-04 02:03:23.732296279 +0000 +@@ -41,8 +41,10 @@ + --enable-icmp \ + --enable-zph-qos \ + --enable-ecap \ ++ --enable-ssl \ + --disable-translation \ + --with-swapdir=/var/spool/squid3 \ ++ --with-open-ssl=/etc/ssl/openssl.cnf \ + --with-logdir=/var/log/squid3 \ + --with-pidfile=/var/run/squid3.pid \ + --with-filedescriptors=65536 \ diff --git a/start_squid.sh b/start_squid.sh index cf78b03..f40612e 100755 --- a/start_squid.sh +++ b/start_squid.sh @@ -1,16 +1,38 @@ #!/bin/bash -# Setup the NAT rule that enables transparent proxying -IPADDR=$(/sbin/ip -o -f inet addr show eth0 | awk '{ sub(/\/.+/,"",$4); print $4 }') -iptables -t nat -A PREROUTING -p tcp --dport 80 -j DNAT --to-destination ${IPADDR}:3128 +function gen-cert() { + cd /etc/squid3/ssl_cert + openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \ + -keyout myCA.pem -out myCA.pem \ + -subj '/CN=squid-ssl/O=NULL/C=AU' + chmod 600 myCA.pem + openssl x509 -in myCA.pem -outform DER -out myCA.der + chown proxy.proxy myCA.pem + return $? +} -# Make sure our cache is setup -chown proxy.proxy /var/spool/squid3 -[ -e /var/spool/squid3/swap.state ] || squid3 -z 2>/dev/null +function start-routing() { + # Setup the NAT rule that enables transparent proxying + IPADDR=$(/sbin/ip -o -f inet addr show eth0 | awk '{ sub(/\/.+/,"",$4); print $4 }') + iptables -t nat -A PREROUTING -p tcp --dport 80 -j DNAT --to-destination ${IPADDR}:3128 + return $? +} + +function init-cache() { + # Make sure our cache is setup + touch /var/log/squid3/access.log /var/log/squid3/cache.log + chown proxy.proxy -R /var/spool/squid3 /var/log/squid3 + [ -e /var/spool/squid3/swap.state ] || squid3 -z 2>/dev/null +} + +gen-cert || exit 1 +start-routing || exit 1 +init-cache + +echo starting server # Run squid and tail the logs squid3 -while true; do - [ -e /var/log/squid3/access.log ] && tail -f /var/log/squid3/access.log - sleep 1 -done +sleep 0.5 +ps aux +tail -f /var/log/squid3/access.log /var/log/squid3/cache.log From 9228b44b00d8404debc7ead46e8e37648eb08c7c Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Wed, 6 Apr 2016 11:15:14 +1000 Subject: [PATCH 02/23] Fixed removal of firewall rules when container IP is unknown --- run.sh | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/run.sh b/run.sh index b6fb3d7..7f9125c 100755 --- a/run.sh +++ b/run.sh @@ -38,22 +38,25 @@ start_routing () { } stop_routing () { - # Remove the appropriate rules - that is, those that mention the IP Address. - set +e - [ "x$IPADDR" != "x" ] && { + # Remove iptables rules. + set +e ip route show table TRANSPROXY | grep -q default \ - && sudo ip route del default table TRANSPROXY - sudo iptables -t mangle -L PREROUTING -n | grep -q 'tcp dpt:80 MARK set 0x1' \ - && sudo iptables -t mangle -D PREROUTING -p tcp --dport 80 \! -s "${IPADDR}" -i docker0 -j MARK --set-mark 1 \ - && sudo iptables -t mangle -D PREROUTING -p tcp --dport 443 \! -s "${IPADDR}" -i docker0 -j MARK --set-mark 1 \ + && sudo ip route del default table TRANSPROXY + while true; do + rule_num=$(sudo iptables -t mangle -L PREROUTING -n --line-numbers \ + | grep -E 'MARK.*172\.17.*tcp \S+ MARK set 0x1' \ + | awk '{print $1}' \ + | head -n1) + [ -z "$rule_num" ] && break + sudo iptables -t mangle -D PREROUTING "$rule_num" + done sudo iptables -t nat -D POSTROUTING -o docker0 -s 172.17.0.0/16 -j ACCEPT 2>/dev/null - } - set -e + set -e } stop () { set +e - sudo docker rm -f ${CONTAINER_NAME} >/dev/null 2>&1 + sudo docker rm -fv ${CONTAINER_NAME} >/dev/null 2>&1 set -e stop_routing } From 6489bc8cb2f7c8063fb29d8d210cff93c2f6ceaa Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Wed, 6 Apr 2016 11:15:43 +1000 Subject: [PATCH 03/23] Fixed configuration of squid's ssl-bump --- Dockerfile | 2 +- squid.conf | 7 +++++-- start_squid.sh | 19 ++++++++----------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 685e77d..c448958 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,6 +41,6 @@ ADD squid.conf /etc/squid3/squid.conf ADD start_squid.sh /usr/local/bin/start_squid.sh VOLUME /var/spool/squid3 -EXPOSE 3128 3129 +EXPOSE 3128 3129 3130 CMD ["/usr/local/bin/start_squid.sh"] diff --git a/squid.conf b/squid.conf index d53dcf4..28d80f2 100644 --- a/squid.conf +++ b/squid.conf @@ -18,9 +18,12 @@ http_access deny manager http_access allow localhost http_access allow all #http_access deny all +ssl_bump none all +ssl_bump server-first all -http_port 3128 intercept -https_port 3129 cert=/etc/squid3/ssl_cert/myCA.pem generate-host-certificates=on intercept ssl-bump +http_port 3128 +http_port 3129 intercept +https_port 3130 cert=/etc/squid3/ssl_cert/ca.pem key=/etc/squid3/ssl_cert/privkey.pem generate-host-certificates=on intercept ssl-bump maximum_object_size 512 MB cache_dir ufs /var/spool/squid3 2024 2 8 diff --git a/start_squid.sh b/start_squid.sh index f40612e..d5bfd98 100755 --- a/start_squid.sh +++ b/start_squid.sh @@ -1,20 +1,22 @@ #!/bin/bash function gen-cert() { - cd /etc/squid3/ssl_cert + pushd /etc/squid3/ssl_cert openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \ - -keyout myCA.pem -out myCA.pem \ + -keyout privkey.pem -out ca.pem \ -subj '/CN=squid-ssl/O=NULL/C=AU' - chmod 600 myCA.pem - openssl x509 -in myCA.pem -outform DER -out myCA.der - chown proxy.proxy myCA.pem + chown proxy.proxy privkey.pem + chmod 600 privkey.pem + openssl x509 -in ca.pem -outform DER -out ca.der + popd return $? } function start-routing() { # Setup the NAT rule that enables transparent proxying IPADDR=$(/sbin/ip -o -f inet addr show eth0 | awk '{ sub(/\/.+/,"",$4); print $4 }') - iptables -t nat -A PREROUTING -p tcp --dport 80 -j DNAT --to-destination ${IPADDR}:3128 + iptables -t nat -A PREROUTING -p tcp --dport 80 -j DNAT --to-destination ${IPADDR}:3129 + iptables -t nat -A PREROUTING -p tcp --dport 443 -j DNAT --to-destination ${IPADDR}:3130 return $? } @@ -29,10 +31,5 @@ gen-cert || exit 1 start-routing || exit 1 init-cache -echo starting server - -# Run squid and tail the logs squid3 -sleep 0.5 -ps aux tail -f /var/log/squid3/access.log /var/log/squid3/cache.log From f52448ac23e09b3c02d24d0f148a107b383d31dc Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Wed, 6 Apr 2016 14:32:50 +1000 Subject: [PATCH 04/23] Made the CA cert available for download from the squid proxy Clients can get a copy of the CA cert by downloading it from http://docker-proxy:3128/squid-internal-static/icons/ca.pem It's a bit of a hack, because it's pretending to an icon, which requires that it be configured as a MIME type so that Squid knows to serve it. --- Dockerfile | 8 +++++--- mime.conf | 5 +++++ squid.conf | 5 ++++- start_squid.sh | 3 +++ 4 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 mime.conf diff --git a/Dockerfile b/Dockerfile index c448958..1fb1b6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,13 +32,15 @@ RUN cd squid3-3.?.? \ && patch -p1 < /root/squid3.patch \ && export NUM_PROCS=`grep -c ^processor /proc/cpuinfo` \ && (dpkg-buildpackage -b -j${NUM_PROCS} || dpkg-buildpackage -b -j${NUM_PROCS}) +COPY mime.conf /root/ RUN dpkg -i \ squid3-common_3.?.?-?ubuntu?.?_all.deb \ squid3_3.?.?-?ubuntu?.?_*.deb \ - && mkdir -p /etc/squid3/ssl_cert + && mkdir -p /etc/squid3/ssl_cert \ + && cat mime.conf >> /usr/share/squid3/mime.conf -ADD squid.conf /etc/squid3/squid.conf -ADD start_squid.sh /usr/local/bin/start_squid.sh +COPY squid.conf /etc/squid3/squid.conf +COPY start_squid.sh /usr/local/bin/start_squid.sh VOLUME /var/spool/squid3 EXPOSE 3128 3129 3130 diff --git a/mime.conf b/mime.conf new file mode 100644 index 0000000..46db037 --- /dev/null +++ b/mime.conf @@ -0,0 +1,5 @@ +# These dummy types configure Squid to serve the certificate as a static file. +# This allows clients to download the certificate from the proxy server. +# The .pem file is the certificate; the .der file is for browsers. +\.pem$ application/x-pem-file1 ca.pem - ascii +download +\.der$ application/x-der-file1 ca.der - image +download diff --git a/squid.conf b/squid.conf index 28d80f2..dd2e1d7 100644 --- a/squid.conf +++ b/squid.conf @@ -18,9 +18,12 @@ http_access deny manager http_access allow localhost http_access allow all #http_access deny all -ssl_bump none all ssl_bump server-first all +# Port 3128 is the HTTP Forwarding port, which allows for serving of error pages +# and icons, and in our case, the local CA cert. The other ports are used to +# intercept regular traffic and cache responses using firewall rules (listed +# elsewhere). http_port 3128 http_port 3129 intercept https_port 3130 cert=/etc/squid3/ssl_cert/ca.pem key=/etc/squid3/ssl_cert/privkey.pem generate-host-certificates=on intercept ssl-bump diff --git a/start_squid.sh b/start_squid.sh index d5bfd98..7b5a009 100755 --- a/start_squid.sh +++ b/start_squid.sh @@ -8,6 +8,9 @@ function gen-cert() { chown proxy.proxy privkey.pem chmod 600 privkey.pem openssl x509 -in ca.pem -outform DER -out ca.der + # Make CA certificate available for download via HTTP Forwarding port + # e.g. GET http://docker-proxy:3128/squid-internal-static/icons/ca.pem + cp `pwd`/ca.* /usr/share/squid3/icons/ popd return $? } From fc285a1db4f4ad55ef51882484a6864f63b11523 Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Wed, 6 Apr 2016 17:00:28 +1000 Subject: [PATCH 05/23] Publishing forwarding proxy port on host This makes it easier for clients to detect the proxy. --- run.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/run.sh b/run.sh index 7f9125c..24f163c 100755 --- a/run.sh +++ b/run.sh @@ -81,8 +81,13 @@ run () { mkdir -p "${CACHEDIR}" # Because we're named, make sure the container doesn't already exist stop - # Run and find the IP for the running container - CID=$(sudo docker run --privileged -d -v "${CACHEDIR}":/var/spool/squid3 --name ${CONTAINER_NAME} ${CONTAINER_NAME}) + # Run and find the IP for the running container. Bind the forward proxy port + # so clients can get the CA certificate. + CID=$(sudo docker run --privileged -d \ + --name ${CONTAINER_NAME} \ + --volume="${CACHEDIR}":/var/spool/squid3 \ + --publish=3128:3128 \ + ${CONTAINER_NAME}) IPADDR=$(sudo docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${CID}) start_routing # Run at console, kill cleanly if ctrl-c is hit From f29fc28b06cceb2361349aa76b7c4cf78daeb114 Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Wed, 6 Apr 2016 17:38:22 +1000 Subject: [PATCH 06/23] Added a test build file that uses the SSL proxy During build, it detects the proxy server and installs the local CA certificate. Then programs like curl can make HTTPS requests that are cached, and they don't report an SSL error. --- test/Dockerfile | 16 ++++++++++++++++ test/detect-proxy.sh | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 test/Dockerfile create mode 100755 test/detect-proxy.sh diff --git a/test/Dockerfile b/test/Dockerfile new file mode 100644 index 0000000..79ed632 --- /dev/null +++ b/test/Dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:14.04 + +MAINTAINER Alex Fraser + +# Run a caching proxy on the host and bind a port to APT_PROXY_PORT to cache +# apt requests. Build with `docker build --build-arg APT_PROXY_PORT=[X] [...]`. +WORKDIR /root +COPY detect-proxy.sh /root/ +RUN export DEBIAN_FRONTEND=noninteractive TERM=linux \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + && ./detect-proxy.sh start + +CMD curl -O https://httpbin.org/get diff --git a/test/detect-proxy.sh b/test/detect-proxy.sh new file mode 100755 index 0000000..ddcae7f --- /dev/null +++ b/test/detect-proxy.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +function download-cert() { + AWK_SPLIT=' + !fout && /^\r?$/ { fout="docker-proxy.pem"; next } + fout { print > fout } + !fout { print } + ' + + HOST_IP=$(route -n | awk '/^0.0.0.0/ {print $2}') + echo -e 'GET /squid-internal-static/icons/ca.pem\r\n' \ + | nc -q -1 "$HOST_IP" 3128 \ + | awk "$AWK_SPLIT" \ + | grep -q 'Server: squid' +} + +download-cert +if [ $? -ne 0 ]; then + echo "No proxy server detected" + exit 0 +fi + +grep -q '\-----BEGIN CERTIFICATE-----' docker-proxy.pem +if [ $? -ne 0 ]; then + echo "Proxy detected" + exit 0 +fi + +echo "SSL-caching proxy server detected. Installing certificate." + +cp docker-proxy.pem /usr/local/share/ca-certificates/docker-proxy.crt +update-ca-certificates From 2fa84c5f54ea50f5cc74f96a382c27a4c91fe73a Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Thu, 7 Apr 2016 11:12:40 +1000 Subject: [PATCH 07/23] Reusing CA certificate Using a volume to persist the CA certificate so it doesn't need to be reinstalled in the clients when the proxy is restarted. --- Dockerfile | 20 ++++++++++++-------- run.sh | 10 +++++++--- start_squid.sh | 21 +++++++++++++-------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1fb1b6e..7c040d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,11 @@ FROM ubuntu:14.04 MAINTAINER Alex Fraser +# Install base dependencies. # Run a caching proxy on the host and bind a port to APT_PROXY_PORT to cache # apt requests. Build with `docker build --build-arg APT_PROXY_PORT=[X] [...]`. +# Not required if you're using a transparent proxy (like the one built by +# this project). WORKDIR /root ARG APT_PROXY_PORT= COPY detect-apt-proxy.sh /root/ @@ -24,25 +27,26 @@ RUN export DEBIAN_FRONTEND=noninteractive TERM=linux \ # rm -rf /var/lib/apt/lists/* \ # /etc/apt/apt.conf.d/30proxy \ +# Customise and build Squid. # It's silly, but run dpkg-buildpackage again if it fails the first time. This # is needed because sometimes the `configure` script is busy when building in # Docker after autoconf sets its mode +x. -COPY squid3.patch /root/ +COPY squid3.patch mime.conf /root/ RUN cd squid3-3.?.? \ && patch -p1 < /root/squid3.patch \ && export NUM_PROCS=`grep -c ^processor /proc/cpuinfo` \ - && (dpkg-buildpackage -b -j${NUM_PROCS} || dpkg-buildpackage -b -j${NUM_PROCS}) -COPY mime.conf /root/ -RUN dpkg -i \ - squid3-common_3.?.?-?ubuntu?.?_all.deb \ - squid3_3.?.?-?ubuntu?.?_*.deb \ + && (dpkg-buildpackage -b -j${NUM_PROCS} \ + || dpkg-buildpackage -b -j${NUM_PROCS}) \ + && DEBIAN_FRONTEND=noninteractive TERM=linux dpkg -i \ + ../squid3-common_3.?.?-?ubuntu?.?_all.deb \ + ../squid3_3.?.?-?ubuntu?.?_*.deb \ && mkdir -p /etc/squid3/ssl_cert \ - && cat mime.conf >> /usr/share/squid3/mime.conf + && cat /root/mime.conf >> /usr/share/squid3/mime.conf COPY squid.conf /etc/squid3/squid.conf COPY start_squid.sh /usr/local/bin/start_squid.sh -VOLUME /var/spool/squid3 +VOLUME /var/spool/squid3 /etc/squid3/ssl_cert EXPOSE 3128 3129 3130 CMD ["/usr/local/bin/start_squid.sh"] diff --git a/run.sh b/run.sh index 24f163c..490c893 100755 --- a/run.sh +++ b/run.sh @@ -4,6 +4,7 @@ # proxy server for docker. CACHEDIR=${CACHEDIR:-/tmp/squid3} +CERTDIR=${CACHEDIR:-/tmp/squid3_cert} CONTAINER_NAME=${CONTAINER_NAME:-docker-proxy} set -e @@ -30,8 +31,10 @@ start_routing () { sudo ip route add default via "${IPADDR}" dev docker0 table TRANSPROXY # Mark packets to port 80 and 443 external, so they route through the new # route table - sudo iptables -t mangle -I PREROUTING -p tcp --dport 80 \! -s "${IPADDR}" -i docker0 -j MARK --set-mark 1 - sudo iptables -t mangle -I PREROUTING -p tcp --dport 443 \! -s "${IPADDR}" -i docker0 -j MARK --set-mark 1 + COMMON_RULES="-t mangle -I PREROUTING -p tcp -i docker0 ! -s ${IPADDR} + -j MARK --set-mark 1" + sudo iptables $COMMON_RULES --dport 80 + sudo iptables $COMMON_RULES --dport 443 # Exemption rule to stop docker from masquerading traffic routed to the # transparent proxy sudo iptables -t nat -I POSTROUTING -o docker0 -s 172.17.0.0/16 -j ACCEPT @@ -78,7 +81,7 @@ terminated () { run () { # Make sure we have a cache dir - if you're running in vbox you should # probably map this through to the host machine for persistence - mkdir -p "${CACHEDIR}" + mkdir -p "${CACHEDIR}" "${CERTDIR}" # Because we're named, make sure the container doesn't already exist stop # Run and find the IP for the running container. Bind the forward proxy port @@ -86,6 +89,7 @@ run () { CID=$(sudo docker run --privileged -d \ --name ${CONTAINER_NAME} \ --volume="${CACHEDIR}":/var/spool/squid3 \ + --volume="${CERTDIR}":/etc/squid3/ssl_cert \ --publish=3128:3128 \ ${CONTAINER_NAME}) IPADDR=$(sudo docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${CID}) diff --git a/start_squid.sh b/start_squid.sh index 7b5a009..e86250c 100755 --- a/start_squid.sh +++ b/start_squid.sh @@ -1,17 +1,22 @@ #!/bin/bash function gen-cert() { - pushd /etc/squid3/ssl_cert - openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \ - -keyout privkey.pem -out ca.pem \ - -subj '/CN=squid-ssl/O=NULL/C=AU' - chown proxy.proxy privkey.pem - chmod 600 privkey.pem - openssl x509 -in ca.pem -outform DER -out ca.der + pushd /etc/squid3/ssl_cert > /dev/null + if [ ! -f ca.pem ]; then + openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes \ + -x509 -keyout privkey.pem -out ca.pem \ + -subj '/CN=docker-proxy/O=NULL/C=AU' + chown proxy.proxy privkey.pem + chmod 600 privkey.pem + openssl x509 -in ca.pem -outform DER -out ca.der + else + echo "Reusing existing certificate" + fi + openssl x509 -sha1 -in ca.pem -noout -fingerprint # Make CA certificate available for download via HTTP Forwarding port # e.g. GET http://docker-proxy:3128/squid-internal-static/icons/ca.pem cp `pwd`/ca.* /usr/share/squid3/icons/ - popd + popd > /dev/null return $? } From d778c012bd6b1c75bde8f14f2550bcb507a4fc7f Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Thu, 7 Apr 2016 11:54:25 +1000 Subject: [PATCH 08/23] Tests now detect proxy Tests now confirm the presence of the proxy server when testing HTTPS. --- test/Dockerfile | 13 ++++++++----- test/test-proxy.sh | 13 +++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) create mode 100755 test/test-proxy.sh diff --git a/test/Dockerfile b/test/Dockerfile index 79ed632..af3e2e3 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -2,15 +2,18 @@ FROM ubuntu:14.04 MAINTAINER Alex Fraser -# Run a caching proxy on the host and bind a port to APT_PROXY_PORT to cache -# apt requests. Build with `docker build --build-arg APT_PROXY_PORT=[X] [...]`. +# Install ca-certificates so we can install the proxy's certificate. curl is +# only needed for running the test, not for installing the certificate. WORKDIR /root -COPY detect-proxy.sh /root/ RUN export DEBIAN_FRONTEND=noninteractive TERM=linux \ && apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ - && ./detect-proxy.sh start + && rm -rf /var/lib/apt/lists/* -CMD curl -O https://httpbin.org/get +# Install the proxy's CA certificate. +COPY detect-proxy.sh test-proxy.sh /root/ +RUN ./detect-proxy.sh start + +CMD /root/test-proxy.sh diff --git a/test/test-proxy.sh b/test/test-proxy.sh new file mode 100755 index 0000000..0710f45 --- /dev/null +++ b/test/test-proxy.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +curl -sS -o /dev/null https://httpbin.org/get 2>&1 || exit 1 + +curl -sS -v -o /dev/null https://httpbin.org/get 2>&1 \ + | grep -q 'X-Cache:' + +if [ $? -ne 0 ]; then + echo "Request succeeded but response was not cached" >&2 + exit 1 +fi + +echo "All tests passed" From 823039b5e43da707d14fe5e4cb2c0aa2ea248c5a Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Thu, 7 Apr 2016 13:11:13 +1000 Subject: [PATCH 09/23] Added a test for using Java to fetch resources It seems that Java, when installed with apt, uses the OS' key store - so there's no need to use Java's keytool. --- test/Dockerfile | 11 +++++++---- test/HttpTest.java | 9 +++++++++ test/detect-proxy.sh | 1 + test/test-proxy.sh | 2 ++ 4 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 test/HttpTest.java diff --git a/test/Dockerfile b/test/Dockerfile index af3e2e3..1cb5554 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -2,18 +2,21 @@ FROM ubuntu:14.04 MAINTAINER Alex Fraser -# Install ca-certificates so we can install the proxy's certificate. curl is -# only needed for running the test, not for installing the certificate. +# Install ca-certificates so we can install the proxy's certificate. curl and +# Java are only needed for running the test, not for installing the +# certificate. WORKDIR /root RUN export DEBIAN_FRONTEND=noninteractive TERM=linux \ && apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ + default-jdk \ && rm -rf /var/lib/apt/lists/* # Install the proxy's CA certificate. -COPY detect-proxy.sh test-proxy.sh /root/ -RUN ./detect-proxy.sh start +COPY detect-proxy.sh test-proxy.sh HttpTest.java /root/ +RUN javac HttpTest.java \ + && ./detect-proxy.sh start CMD /root/test-proxy.sh diff --git a/test/HttpTest.java b/test/HttpTest.java new file mode 100644 index 0000000..7575e22 --- /dev/null +++ b/test/HttpTest.java @@ -0,0 +1,9 @@ +import java.io.*; +import java.net.*; + +public class HttpTest { + public static void main(String[] args) throws Exception { + URL url = new URL(args[0]); + url.openConnection().getContent(); + } +} diff --git a/test/detect-proxy.sh b/test/detect-proxy.sh index ddcae7f..92db621 100755 --- a/test/detect-proxy.sh +++ b/test/detect-proxy.sh @@ -28,5 +28,6 @@ fi echo "SSL-caching proxy server detected. Installing certificate." +# Install CA cert into OS key store cp docker-proxy.pem /usr/local/share/ca-certificates/docker-proxy.crt update-ca-certificates diff --git a/test/test-proxy.sh b/test/test-proxy.sh index 0710f45..4b1957b 100755 --- a/test/test-proxy.sh +++ b/test/test-proxy.sh @@ -10,4 +10,6 @@ if [ $? -ne 0 ]; then exit 1 fi +java HttpTest https://httpbin.org/get || exit 1 + echo "All tests passed" From 86b17be4acec87de6c2e9db650f16026ec5f2c92 Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Thu, 7 Apr 2016 15:31:51 +1000 Subject: [PATCH 10/23] Added an option to not redirect HTTPS This is so that by default the proxy behaves similarly to how it used to before SSL support was added. --- run.sh | 14 +++++++++++++- test/test-proxy.sh | 2 ++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/run.sh b/run.sh index 490c893..0df9560 100755 --- a/run.sh +++ b/run.sh @@ -6,6 +6,11 @@ CACHEDIR=${CACHEDIR:-/tmp/squid3} CERTDIR=${CACHEDIR:-/tmp/squid3_cert} CONTAINER_NAME=${CONTAINER_NAME:-docker-proxy} +if [ "$1" = 'ssl' ]; then + WITH_SSL=yes +else + WITH_SSL=no +fi set -e @@ -33,8 +38,15 @@ start_routing () { # route table COMMON_RULES="-t mangle -I PREROUTING -p tcp -i docker0 ! -s ${IPADDR} -j MARK --set-mark 1" + echo "Redirecting HTTP to docker-proxy" sudo iptables $COMMON_RULES --dport 80 - sudo iptables $COMMON_RULES --dport 443 + if [ "$WITH_SSL" = 'yes' ]; then + echo "Redirecting HTTPS to docker-proxy" + sudo iptables $COMMON_RULES --dport 443 + else + echo "Not redirecting HTTPS. To enable, re-run with the argument 'ssl'" + echo "CA certificate will be generated anyway, but it won't be used" + fi # Exemption rule to stop docker from masquerading traffic routed to the # transparent proxy sudo iptables -t nat -I POSTROUTING -o docker0 -s 172.17.0.0/16 -j ACCEPT diff --git a/test/test-proxy.sh b/test/test-proxy.sh index 4b1957b..b8c3148 100755 --- a/test/test-proxy.sh +++ b/test/test-proxy.sh @@ -10,6 +10,8 @@ if [ $? -ne 0 ]; then exit 1 fi +# Test with Java too, because it may get its certificates from a different key +# store. java HttpTest https://httpbin.org/get || exit 1 echo "All tests passed" From d2d4320299ebee0937d772a9c38cccd3b3c105ce Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Thu, 7 Apr 2016 15:52:52 +1000 Subject: [PATCH 11/23] Updated documentation to explain how to use HTTPS proxying --- README.md | 57 +++++++++++++++++++++++++++++++++++--------- test/detect-proxy.sh | 2 ++ 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 6b2976b..d9c29f4 100644 --- a/README.md +++ b/README.md @@ -5,24 +5,38 @@ Transparent proxy for docker containers, run in a docker container - https://hub ## Instructions for Use +Build first with: + +``` +sudo docker build -t docker-proxy . +``` + +Then run with: + +``` +./run.sh +``` + The "run.sh" script will run the container and setup the appropriate iptables and ip routing rules. -NOTE: This project is _not_ designed to be run with a simple "docker run" - it -requires the "run.sh" script to be run on the docker host, so it can adjust the +NOTE: This project is _not_ designed to be run with a simple `docker run` - it +requires the `run.sh` script to be run on the docker host, so it can adjust the routing rules. You will need to check this code out from https://github.com/silarsis/docker-proxy/ -and run the "run.sh" script on the host (for OS X, that's on your boot2docker or +and run the `run.sh` script on the host (for OS X, that's on your boot2docker or similar host). ## Overview -The run.sh script will fire up a docker container running squid, with -appropriate iptables rules in place for transparent proxying. It will also +The `run.sh` script will fire up a docker container running squid, with +appropriate firewall rules in place for transparent proxying. It will also configure port-based routing on the main host such that any traffic from a -docker container to port 80 routes via the transparent proxy container. +docker container to port 80 routes via the transparent proxy container. It +requires `sudo` access to perform the firewall changes, and it will prompt you +for your password as appropriate. -The run.sh script is designed to run in the foreground, because when the +The `run.sh` script is designed to run in the foreground, because when the container terminates it needs to remove the rules that were redirecting the traffic. @@ -30,10 +44,31 @@ If you want to see squid in operation, you can (in another terminal) attach to the squid container - it is tailing the access log, so will show a record of requests made. -## Notes +## HTTPS Support -* This script assumes you have "sudo" access to perform the iptables changes +The proxy server supports HTTPS caching via Squid's [SSL Bump] feature. To +enable it, start with: + +``` +./run.sh ssl +``` + +The server will decrypt traffic from the server and encrypt it again using its +own root certificate. HTTPS connections from your other Docker containers will +fail until you install the root certificate. You can install it by running the +[`detect-proxy.sh`] script in your container or in your project's +`Dockerfile`. See [`test/Dockerfile`] for an example. + +[SSL Bump]: http://wiki.squid-cache.org/Features/SslBump +[`detect-proxy.sh`]: test/detect-proxy.sh +[`test/Dockerfile`]: test/Dockerfile + +## Notes -* the format of the run.sh script is a bit strange - that's because it's also used as part of a larger framework on my own machine. +This proxy configuration is intended to be used solely to speed +up development of Docker applications. **Do not** attempt to use this to +eavesdrop on other people's connections. -* There exists a real possibility this script will break your iptables or ip rules in some unexpected way. I've tried to be reasonably tidy and careful, but be aware that if things go wrong, the potential exists for all containers to lose the ability to download anything. +There exists a real possibility this script will break your iptables or ip +rules in some unexpected way. Be aware that if things go wrong, the potential +exists for all containers to lose the ability to download anything. diff --git a/test/detect-proxy.sh b/test/detect-proxy.sh index 92db621..f6326eb 100755 --- a/test/detect-proxy.sh +++ b/test/detect-proxy.sh @@ -1,6 +1,8 @@ #!/bin/bash function download-cert() { + # The proxy server hosts the root certificate over HTTP, on a port that is + # published on the host. AWK_SPLIT=' !fout && /^\r?$/ { fout="docker-proxy.pem"; next } fout { print > fout } From c03493b638a40f1f3e898d00749bcb14aada3b13 Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Thu, 7 Apr 2016 15:57:12 +1000 Subject: [PATCH 12/23] Removed URLs to original repository It doesn't make sense to keep these URLs in the repository: they immediately become invalid when a fork is made, and Docker Hub is not useful because the image doesn't include run.sh. --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d9c29f4..3f2dfab 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ -docker-proxy -============ +# docker-proxy -Transparent proxy for docker containers, run in a docker container - https://hub.docker.com/r/silarsis/docker-proxy/ +Transparent caching proxy server for docker containers, run in a docker +container. It can speed up the dependency-fetching part of your application +build process. ## Instructions for Use @@ -22,8 +23,7 @@ and ip routing rules. NOTE: This project is _not_ designed to be run with a simple `docker run` - it requires the `run.sh` script to be run on the docker host, so it can adjust the -routing rules. You will need to check this code out from -https://github.com/silarsis/docker-proxy/ +routing rules. You will need to check this code out and run the `run.sh` script on the host (for OS X, that's on your boot2docker or similar host). From 9e22a41725021e5f770be760e869bd418951bbf4 Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Thu, 7 Apr 2016 17:03:48 +1000 Subject: [PATCH 13/23] Minor documentation clarification --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3f2dfab..9c82e57 100644 --- a/README.md +++ b/README.md @@ -55,9 +55,15 @@ enable it, start with: The server will decrypt traffic from the server and encrypt it again using its own root certificate. HTTPS connections from your other Docker containers will -fail until you install the root certificate. You can install it by running the -[`detect-proxy.sh`] script in your container or in your project's -`Dockerfile`. See [`test/Dockerfile`] for an example. +fail until you install the root certificate. To install it: + + 1. Install the `ca-certificates` package (Debian/Ubuntu images) + 2. Run [`detect-proxy.sh`] + +Those steps can be performed in a running container (for testing), or you can +add them to your `Dockerfile`. `detect-proxy.sh` can be run after you install +your OS packages with apt, because apt shouldn't need HTTPS. See +[`test/Dockerfile`] for an example. [SSL Bump]: http://wiki.squid-cache.org/Features/SslBump [`detect-proxy.sh`]: test/detect-proxy.sh From 1194d00a36c59f47ba7e3f613c4c3ffccf51fba9 Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Thu, 7 Apr 2016 17:14:25 +1000 Subject: [PATCH 14/23] Added a note about add-apt-repository requiring HTTPS --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9c82e57..4ebb195 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,8 @@ fail until you install the root certificate. To install it: Those steps can be performed in a running container (for testing), or you can add them to your `Dockerfile`. `detect-proxy.sh` can be run after you install -your OS packages with apt, because apt shouldn't need HTTPS. See +your OS packages with apt, because apt shouldn't need HTTPS. However, adding +PPAs with `add-apt-repository` will fail until the certificate is installed. See [`test/Dockerfile`] for an example. [SSL Bump]: http://wiki.squid-cache.org/Features/SslBump From 2d75e3fcaa0113c97900d8682352fd54cfb31f67 Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Mon, 11 Apr 2016 10:07:29 +1000 Subject: [PATCH 15/23] Added docs for testing the proxy. --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 4ebb195..43921a8 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,15 @@ your OS packages with apt, because apt shouldn't need HTTPS. However, adding PPAs with `add-apt-repository` will fail until the certificate is installed. See [`test/Dockerfile`] for an example. +To test HTTPS support, do this in another console after starting the proxy: + +``` +cd test +sudo docker build -t test-proxy . +sudo docker run --rm test-proxy +# Should print "All tests passed" +``` + [SSL Bump]: http://wiki.squid-cache.org/Features/SslBump [`detect-proxy.sh`]: test/detect-proxy.sh [`test/Dockerfile`]: test/Dockerfile From 68cf3a37170c79ebe7c2090a2438933c69236a14 Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Mon, 11 Apr 2016 14:40:11 +1000 Subject: [PATCH 16/23] Using aufs, and caching .pom files Maven/Gradle try to fetch .pom files in addition to .jars. Also, jcentre sets a must-revalidate header that needs to be ignored. --- squid.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/squid.conf b/squid.conf index dd2e1d7..ab46a5d 100644 --- a/squid.conf +++ b/squid.conf @@ -29,7 +29,7 @@ http_port 3129 intercept https_port 3130 cert=/etc/squid3/ssl_cert/ca.pem key=/etc/squid3/ssl_cert/privkey.pem generate-host-certificates=on intercept ssl-bump maximum_object_size 512 MB -cache_dir ufs /var/spool/squid3 2024 2 8 +cache_dir aufs /var/spool/squid3 2024 2 8 cache_mem 200 MB maximum_object_size_in_memory 100 MB cache_replacement_policy heap LFUDA @@ -38,5 +38,6 @@ refresh_pattern Packages\.bz2$ 0 20% 4320 refresh-ims refresh_pattern Sources\.bz2$ 0 20% 4320 refresh-ims refresh_pattern Release\.gpg$ 0 20% 4320 refresh-ims refresh_pattern Release$ 0 20% 4320 refresh-ims -refresh_pattern -i .(udeb|tar.gz|deb|rpm|exe|zip|tar|tgz|bz2|ram|rar|bin|jar)$ 129600 100% 129600 override-expire ignore-no-cache ignore-no-store +refresh_pattern -i .(udeb|tar.gz|deb|rpm|exe|zip|tar|tgz|bz2|ram|rar|bin)$ 129600 100% 129600 override-expire ignore-no-cache ignore-no-store +refresh_pattern -i .(jar|pom)$ 129600 100% 129600 override-expire ignore-no-cache ignore-no-store ignore-must-revalidate refresh_pattern . 0 20% 4320 From 072542651d28870ca3e1226d08dda8c1a8709760 Mon Sep 17 00:00:00 2001 From: Jin Park Date: Fri, 15 Apr 2016 12:51:46 +1000 Subject: [PATCH 17/23] add command to execute squid in MAC OSX --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 43921a8..09b57dd 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,13 @@ Then run with: ./run.sh ``` +For Max(OSX): +``` +docker-machine scp ./run.sh default:/home/docker/run.sh +docker-machine ssh default +sh ./run.sh +``` + The "run.sh" script will run the container and setup the appropriate iptables and ip routing rules. From b4490b51ce6d8cdafb04fa915018313d256f7acc Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Wed, 27 Apr 2016 10:44:05 +1000 Subject: [PATCH 18/23] Various documentation improvements Added more details about starting the proxy, to serve as a more in-depth example. Also improved some grammar, and added a note about SSL not working out of the box for some programs. --- README.md | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 09b57dd..e2b92ca 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # docker-proxy -Transparent caching proxy server for docker containers, run in a docker +Transparent caching proxy server for Docker containers, run in a Docker container. It can speed up the dependency-fetching part of your application build process. ## Instructions for Use -Build first with: +First check out the code. Then build with: ``` sudo docker build -t docker-proxy . @@ -18,38 +18,42 @@ Then run with: ./run.sh ``` -For Max(OSX): +The script will start the container and set up the appropriate +routing rules. Your other Docker containers will automatically use +the proxy, whether or not they were already running. When you are finished, +just press CtrlC to stop the proxy. + +NOTE: This project is _not_ designed to be run with a simple `docker run` - it +requires `run.sh` to be run on the docker host, so it can adjust the +routing rules. You will need to check this code out +and run `run.sh` on the host. For OS X, that's on your [boot2docker], +Docker Machine or similar host). To start under [Docker Machine] on OS X: + ``` -docker-machine scp ./run.sh default:/home/docker/run.sh +docker-machine scp run.sh default:/home/docker/run.sh docker-machine ssh default sh ./run.sh ``` -The "run.sh" script will run the container and setup the appropriate iptables -and ip routing rules. - -NOTE: This project is _not_ designed to be run with a simple `docker run` - it -requires the `run.sh` script to be run on the docker host, so it can adjust the -routing rules. You will need to check this code out -and run the `run.sh` script on the host (for OS X, that's on your boot2docker or -similar host). +[boot2docker]: http://boot2docker.io/ +[Docker Machine]: https://docs.docker.com/machine/get-started/ ## Overview -The `run.sh` script will fire up a docker container running squid, with +`run.sh` will fire up a Docker container running Squid, with appropriate firewall rules in place for transparent proxying. It will also configure port-based routing on the main host such that any traffic from a -docker container to port 80 routes via the transparent proxy container. It +Docker container to port 80 routes via the transparent proxy container. It requires `sudo` access to perform the firewall changes, and it will prompt you for your password as appropriate. -The `run.sh` script is designed to run in the foreground, because when the +`run.sh` is designed to run in the foreground, because when the container terminates it needs to remove the rules that were redirecting the traffic. -If you want to see squid in operation, you can (in another terminal) attach -to the squid container - it is tailing the access log, so will show a record -of requests made. +If you want to see Squid in operation, you can (in another terminal) attach +to the `docker-proxy` container - it is tailing the access log, so will show a +record of requests made. ## HTTPS Support @@ -73,6 +77,9 @@ your OS packages with apt, because apt shouldn't need HTTPS. However, adding PPAs with `add-apt-repository` will fail until the certificate is installed. See [`test/Dockerfile`] for an example. +Some programs don't use the OS's primary key store, such as `npm` and `pip`. +You may need to take extra steps for those programs. + To test HTTPS support, do this in another console after starting the proxy: ``` @@ -92,6 +99,6 @@ This proxy configuration is intended to be used solely to speed up development of Docker applications. **Do not** attempt to use this to eavesdrop on other people's connections. -There exists a real possibility this script will break your iptables or ip +There exists a real possibility this script will break your `iptables` or `ip` rules in some unexpected way. Be aware that if things go wrong, the potential exists for all containers to lose the ability to download anything. From 64577770fb9cd3a77ee092abb6eca30e86dd038f Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Wed, 27 Apr 2016 11:04:30 +1000 Subject: [PATCH 19/23] Removed old apt-proxy-ng detection script. --- Dockerfile | 12 ++---------- detect-apt-proxy.sh | 20 -------------------- 2 files changed, 2 insertions(+), 30 deletions(-) delete mode 100755 detect-apt-proxy.sh diff --git a/Dockerfile b/Dockerfile index 7c040d5..e1a35e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,11 @@ FROM ubuntu:14.04 -MAINTAINER Alex Fraser +MAINTAINER Kevin Littlejohn , \ + Alex Fraser # Install base dependencies. -# Run a caching proxy on the host and bind a port to APT_PROXY_PORT to cache -# apt requests. Build with `docker build --build-arg APT_PROXY_PORT=[X] [...]`. -# Not required if you're using a transparent proxy (like the one built by -# this project). WORKDIR /root -ARG APT_PROXY_PORT= -COPY detect-apt-proxy.sh /root/ RUN export DEBIAN_FRONTEND=noninteractive TERM=linux \ - && ./detect-apt-proxy.sh ${APT_PROXY_PORT} no \ && apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ @@ -24,8 +18,6 @@ RUN export DEBIAN_FRONTEND=noninteractive TERM=linux \ ssl-cert \ && apt-get source -y squid3 squid-langpack \ && apt-get build-dep -y squid3 squid-langpack -# rm -rf /var/lib/apt/lists/* \ -# /etc/apt/apt.conf.d/30proxy \ # Customise and build Squid. # It's silly, but run dpkg-buildpackage again if it fails the first time. This diff --git a/detect-apt-proxy.sh b/detect-apt-proxy.sh deleted file mode 100755 index 8db0031..0000000 --- a/detect-apt-proxy.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# If the host is running a web proxy, use it for apt. -# Adapted from https://gist.github.com/dergachev/8441335 - -APT_PROXY_PORT=$1 -PROXY_LAUNCHPAD=${2:-no} - -HOST_IP=$(route -n | awk '/^0.0.0.0/ {print $2}') -nc -z "$HOST_IP" ${APT_PROXY_PORT} - -if [ $? -eq 0 ]; then - echo "Acquire::http::Proxy \"http://$HOST_IP:$APT_PROXY_PORT\";" >> /etc/apt/apt.conf.d/30proxy - if [ "$PROXY_LAUNCHPAD" = yes ]; then - echo "Acquire::http::Proxy::ppa.launchpad.net DIRECT;" >> /etc/apt/apt.conf.d/30proxy - fi - echo "Using host's apt proxy" -else - echo "No apt proxy detected on Docker host" -fi From 72214785a5a0dcc93ce0bb8c21ad18aee8f39023 Mon Sep 17 00:00:00 2001 From: Alex Fraser Date: Mon, 2 May 2016 09:47:38 +1000 Subject: [PATCH 20/23] Fixed run.sh options, and persisting cache and key by default The SSL certificate directory was erroneously being set to use the cache directory ($CACHEDIR). Also, the default directories were in /tmp. This meant that every time the host computer was restarted, the cache - and more importantly, the CA certificate - was destroyed. Images that had been built using the old CA certificate would no longer be able to use the proxy, so it's important that the CA certificate at least is persisted. --- run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.sh b/run.sh index 0df9560..d997f65 100755 --- a/run.sh +++ b/run.sh @@ -3,8 +3,8 @@ # Script to maintain ip rules on the host when starting up a transparent # proxy server for docker. -CACHEDIR=${CACHEDIR:-/tmp/squid3} -CERTDIR=${CACHEDIR:-/tmp/squid3_cert} +CACHEDIR=${CACHEDIR:-/var/lib/docker-proxy/cache} +CERTDIR=${CERTDIR:-/var/lib/docker-proxy/ssl} CONTAINER_NAME=${CONTAINER_NAME:-docker-proxy} if [ "$1" = 'ssl' ]; then WITH_SSL=yes From b220ddfda520a6a10cb1a09cda7ea5839aae9d62 Mon Sep 17 00:00:00 2001 From: hoijui Date: Sun, 1 Oct 2017 11:22:26 +0200 Subject: [PATCH 21/23] make source package repo available again [fix] We need it for installing `squid3` sources. All credits go to @Malvineous, see github issue #21: https://github.com/silarsis/docker-proxy/issues/21 --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index e1a35e6..a784665 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ MAINTAINER Kevin Littlejohn , \ # Install base dependencies. WORKDIR /root +RUN sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list RUN export DEBIAN_FRONTEND=noninteractive TERM=linux \ && apt-get update \ && apt-get install -y --no-install-recommends \ From 3791f3bc16090de4b2a7f7a4b2f46ff0bfe8013d Mon Sep 17 00:00:00 2001 From: Adam Nielsen Date: Sun, 17 Jun 2018 17:30:37 +1000 Subject: [PATCH 22/23] Use wildcards for filenames so double-digit version numbers still work --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index a784665..7115d36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,14 +25,14 @@ RUN export DEBIAN_FRONTEND=noninteractive TERM=linux \ # is needed because sometimes the `configure` script is busy when building in # Docker after autoconf sets its mode +x. COPY squid3.patch mime.conf /root/ -RUN cd squid3-3.?.? \ +RUN cd squid3-3.* \ && patch -p1 < /root/squid3.patch \ && export NUM_PROCS=`grep -c ^processor /proc/cpuinfo` \ && (dpkg-buildpackage -b -j${NUM_PROCS} \ || dpkg-buildpackage -b -j${NUM_PROCS}) \ && DEBIAN_FRONTEND=noninteractive TERM=linux dpkg -i \ - ../squid3-common_3.?.?-?ubuntu?.?_all.deb \ - ../squid3_3.?.?-?ubuntu?.?_*.deb \ + ../squid3-common_3.*_all.deb \ + ../squid3_3.*.deb \ && mkdir -p /etc/squid3/ssl_cert \ && cat /root/mime.conf >> /usr/share/squid3/mime.conf From 54b68044fdde1e270250a15ceb62beb687444293 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 29 Mar 2022 04:03:59 +0000 Subject: [PATCH 23/23] fix: Dockerfile to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-UBUNTU1404-OPENSSL-1049144 - https://snyk.io/vuln/SNYK-UBUNTU1404-OPENSSL-2426359 - https://snyk.io/vuln/SNYK-UBUNTU1404-OPENSSL-2426359 - https://snyk.io/vuln/SNYK-UBUNTU1404-PYTHON34-589946 - https://snyk.io/vuln/SNYK-UBUNTU1404-SUDO-1065770 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7115d36..d91e2a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:14.04 +FROM ubuntu:trusty-20190515 MAINTAINER Kevin Littlejohn , \ Alex Fraser