diff --git a/.appveyor.yml b/.appveyor.yml
new file mode 100644
index 000000000..740e2f11c
--- /dev/null
+++ b/.appveyor.yml
@@ -0,0 +1,89 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-24 16:19:35 +0000 (Mon, 24 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# A p p V e y o r C I
+# ============================================================================ #
+
+# https://www.appveyor.com/docs/appveyor-yml/
+
+image: Ubuntu
+
+# workaround for default JDK9 have old CA certs:
+#
+# https://github.com/appveyor/ci/issues/3833
+#
+# https://www.appveyor.com/docs/getting-started-with-appveyor-for-linux/#configuring-language-stack
+#
+stack: jdk 15
+
+skip_commits:
+ files:
+ - docs/*
+ - '**/*.md'
+
+# https://www.appveyor.com/docs/how-to/ssh-to-build-worker/
+environment:
+ APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== hari@anotherdimension
+
+# enable SSH session accessible via my public key
+#init:
+# - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -
+
+# more useful at end to leverage .appveyor.yml tweaks like disabling broken mssql repo/dependencies, checking out project and building the core stuff happen first so we don't have to do all that manually in SSH session
+on_finish:
+ # set this in Settings -> Environment dynamically instead of here
+ #- sh: export APPVEYOR_SSH_BLOCK=true
+ #
+ # workaround for https://github.com/appveyor/ci/issues/3373
+ # and https://github.com/appveyor/ci/issues/3384
+ #
+ # has since been added to AppVeyor's own scripts:
+ #
+ # https://github.com/appveyor/ci/pull/3385
+ #
+ #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/install/install_openssh.sh' | bash -e -
+ #
+ # https://www.appveyor.com/docs/how-to/ssh-to-build-worker/
+ - sh: if [ "$APPVEYOR_SSH_BLOCK" = true ]; then curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -; fi
+
+install:
+ # workaround for:
+ # Some packages could not be installed. This may mean that you have
+ # requested an impossible situation or if you are using the unstable
+ # distribution that some required packages have not yet been created
+ # or been moved out of Incoming.
+ # The following information may help to resolve the situation:
+ #
+ # The following packages have unmet dependencies:
+ # mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed
+ # E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages.
+ # DevOps-Python-tools/Makefile.in:272: recipe for target 'apt-packages' failed
+ # make[2]: *** [apt-packages] Error 123
+ # make[2]: Leaving directory '/home/appveyor/projects/pylib'
+ # DevOps-Python-tools/Makefile.in:212: recipe for target 'system-packages' failed
+ #
+ # adding "|| :" to the end of these commands causes them to be silently ignored!
+ - sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list
+ - sudo apt purge -yq --allow-change-held-packages mssql-server
+ # this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411
+ #- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages
+ - setup/ci_bootstrap.sh
+ - make
+
+test_script:
+ - make test
+
+build: off
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
new file mode 100644
index 000000000..eb0d641fd
--- /dev/null
+++ b/.buildkite/pipeline.yml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-03-13 21:10:39 +0000 (Fri, 13 Mar 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# B u i l d K i t e C I
+# ============================================================================ #
+
+# BuildKite Pipeline
+#
+# add this command to the UI and it will read the rest of the steps from here:
+#
+# - command: buildkite-agent pipeline upload
+
+# Yaml Anchors workaround to BuildKite's lack of global retries configuration - credit to Jason @ BuildKite for this workaround:
+ #
+# https://forum.buildkite.community/t/reschedule-builds-on-other-agents-rather-than-fail-builds-when-agents-time-out-or-are-killed-machine-shut-down-or-put-to-sleep/1388/5
+#
+anchors:
+ std_retries: &std_retries
+ retry:
+ automatic:
+ - exit_status: -1 # Agent was lost
+ limit: 2
+ - exit_status: 255 # Forced agent shutdown
+ limit: 2
+
+steps:
+ - command: setup/ci_bootstrap.sh
+ label: ci bootstrap
+ timeout: 30 # brew can take 10 mins just to do a brew update
+ branches: master
+ <<: [*std_retries]
+ - wait
+ - command: make init
+ label: init
+ timeout: 2
+ branches: master
+ <<: [*std_retries]
+ - wait
+ - command: make ci
+ label: build
+ timeout: 60
+ branches: master
+ <<: [*std_retries]
+ - wait
+ - command: make test
+ label: test
+ timeout: 120
+ branches: master
+ <<: [*std_retries]
diff --git a/.checkov.yaml b/.checkov.yaml
new file mode 100644
index 000000000..8962e2283
--- /dev/null
+++ b/.checkov.yaml
@@ -0,0 +1,48 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-02-21 16:53:29 +0000 (Mon, 21 Feb 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C h e c k o v c o n f i g
+# ============================================================================ #
+
+# https://github.com/bridgecrewio/checkov#configuration-using-a-config-file
+#
+# This is not well documented but the fields seem to be the same as:
+#
+# checkov --help
+#
+# See master template at:
+#
+# https://github.com/HariSekhon/Templates/blob/master/.checkov.yaml
+
+---
+compact: true
+directory:
+ - .
+skip-path:
+ - bash-tools
+ - pylib
+ - sql
+ - templates
+docker-image: harisekhon/pytools
+download-external-modules: true # without this gets lots of annoying warning lines such as '2022-02-22 16:14:40,180 [MainThread ] [WARNI] Failed to download module x/y/z:n.n.n'
+framework:
+ - all
+no-guide: true
+output: cli
+quiet: true
+repo-id: HariSekhon/DevOps-Python-tools # what to report to Bridgecrew Cloud - without this gets annoying duplicate repos such as 'harisekhon_cli_repo/pytools'
+skip-suppressions: true
+soft-fail: true
diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 000000000..312bf1020
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,54 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-23 23:30:14 +0000 (Sun, 23 Feb 2020)
+# Original: H1 2016 (Circle CI 1.x)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C i r c l e C I
+# ============================================================================ #
+
+# Master Template with more advanced config:
+#
+# https://github.com/HariSekhon/Templates/blob/master/circleci_config.yml
+
+# Reference:
+#
+# https://circleci.com/docs/2.0/configuration-reference
+
+version: 2.1
+
+workflows:
+ version: 2
+ workflow:
+ jobs:
+ - build
+
+jobs:
+ build:
+ docker:
+ - image: cimg/base:2024.12
+ resource_class: small
+ steps:
+ # CLI is too old - config validate breaks in test - install new version to fix
+ # doesn't work - existing version is too old to update
+ #- run: circleci update
+ - run: |
+ curl -sSLf https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash
+ - checkout
+ #- setup_remote_docker:
+ # version: 20.10.11
+ - run: setup/ci_bootstrap.sh
+ - run: make init
+ - run: make
+ - run: make test
diff --git a/.cirrus.yml b/.cirrus.yml
new file mode 100644
index 000000000..471dc55ba
--- /dev/null
+++ b/.cirrus.yml
@@ -0,0 +1,31 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-24 16:55:36 +0000 (Mon, 24 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C i r r u s C I
+# ============================================================================ #
+
+# https://cirrus-ci.org/guide/writing-tasks/
+
+container:
+ image: ubuntu:18.04
+
+task:
+ env:
+ TMPDIR: /var/tmp
+ script:
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci test
diff --git a/.drone.yml b/.drone.yml
new file mode 100644
index 000000000..d105ab57a
--- /dev/null
+++ b/.drone.yml
@@ -0,0 +1,49 @@
+---
+# XXX: putting this separator further down with code causes a parsing bug in drone lint
+#
+# Author: Hari Sekhon
+# Date: 2020-02-29 12:05:52 +0000 (Sat, 29 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# D r o n e C I
+# ============================================================================ #
+
+# https://docs.drone.io/quickstart/cli/
+#
+# https://docs.drone.io/cli/install/
+#
+# brew install drone-cli
+#
+# cd to this directory
+#
+# drone exec [--pipeline default] [--include=thisstep] [--exclude=thatstep]
+
+kind: pipeline
+type: docker
+name: default
+
+steps:
+ - name: build
+ image: ubuntu:18.04
+ #environment:
+ # DEBUG: 1
+ commands:
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci
+ - make test
+
+trigger:
+ branch:
+ - master
diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 000000000..462f53206
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,85 @@
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2015-10-31 19:04:34 +0000 (Sat, 31 Oct 2015)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# http://EditorConfig.org
+
+# stop recursing upwards for other .editorconfig files
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+indent_size = 4
+indent_style = space
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.go]
+indent_size = 4
+indent_style = tab
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[Makefile]
+indent_size = 4
+indent_style = tab
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[{*.md,*.hcl,*.tf,*.tfvars}]
+indent_size = 2
+indent_style = space
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.yml,*.yaml]
+indent_size = 2
+indent_style = space
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[.*]
+indent_size = 4
+indent_style = space
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+# ============================================================================ #
+# Older Stuff, don't think I use this any more
+# ============================================================================ #
+
+# Matches multiple files with brace expansion notation
+# Set default charset
+#[*.{js,py}]
+#charset = utf-8
+
+# Indentation override for all JS under lib directory
+#[lib/**.js]
+#indent_style = space
+#indent_size = 2
+
+# Matches the exact files either package.json or .travis.yml
+#[{package.json,.travis.yml}]
+#indent_style = space
+#indent_size = 2
+
+#[*.xml]
+#indent_style = space
+#indent_size = 2
diff --git a/.envrc b/.envrc
new file mode 100644
index 000000000..e4488d51f
--- /dev/null
+++ b/.envrc
@@ -0,0 +1,214 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: Mon Feb 22 17:42:01 2021 +0000
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# D i r E n v
+# ============================================================================ #
+
+# https://direnv.net/man/direnv-stdlib.1.html
+
+# See Also:
+#
+# .envrc-aws
+# .envrc-gcp
+# .envrc-kubernetes
+
+# direnv stdlib - loads .envrc from parent dir up to /
+#
+# useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc.
+#
+# bypasses security authorization though - use with care
+#source_up
+#
+# source_up must be loaded before set -u otherwise gets this error:
+#
+# direnv: loading .envrc
+# /bin/bash: line 226: $1: unbound variable
+#
+# source_up causes this error is up .envrc is found in parent directories:
+#
+# direnv: No ancestor .envrc found
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+src="$(readlink -f "${BASH_SOURCE[0]}")"
+srcdir="$(cd "$(dirname "$src")" && pwd)"
+
+# ============================================================================ #
+# P r e - C o m m i t
+# ============================================================================ #
+
+# Automatically install Pre-Commit Git hooks if not already present
+
+if ! type -P pre-commit &>/dev/null; then
+ if uname -s | grep -q Darwin &&
+ type -P brew &>/dev/null; then
+ echo
+ echo "Pre-commit is not installed - installing now using Homebrew..."
+ echo
+ brew install pre-commit
+ echo
+ elif type -P pip &>/dev/null; then
+ echo
+ echo "Pre-commit is not installed - installing now using Pip..."
+ echo
+ pip install pre-commit
+ fi
+fi
+
+if [ -f .pre-commit-config.yaml ] &&
+ type -P pre-commit &>/dev/null &&
+ git rev-parse --is-inside-work-tree &>/dev/null; then
+ hook="$(git rev-parse --show-toplevel)/.git/hooks/pre-commit"
+ if [ -L "$hook" ]; then
+ echo "Detected symlink hook: "
+ echo
+ ls -l "$hook"
+ echo
+ echo "Removing"
+ rm -f "$hook"
+ fi
+ if ! [ -f "$hook" ]; then
+ echo
+ echo "Pre-commit hook is not installed in local Git repo checkout - installing now..."
+ echo
+ pre-commit install
+ fi
+fi
+
+# ============================================================================ #
+# D o c k e r C o m p o s e
+# ============================================================================ #
+
+export COMPOSE_PROJECT_NAME="DevOps-Python-tools"
+
+# ============================================================================ #
+# G i t H u b
+# ============================================================================ #
+
+#export GITHUB_ORGANIZATION=HariSekhon
+
+# ============================================================================ #
+# A n s i b l e
+# ============================================================================ #
+
+# use the local repo's ansible.cfg rather than:
+#
+# $PWD/ansible.cfg
+# ~/.ansible.cfg
+# /etc/ansible/ansible.cfg
+#
+# set this in project repos to ensure user environment ANSIBLE_CONFIG doesn't get used
+#export ANSIBLE_CONFIG="/path/to/ansible.cfg"
+
+# ============================================================================ #
+# C l o u d f l a r e
+# ============================================================================ #
+
+#export CLOUDFLARE_EMAIL=hari@...
+#export CLOUDFLARE_API_KEY=... # generate here: https://dash.cloudflare.com/profile/api-tokens
+#export CLOUDFLARE_TOKEN=... # used by cloudflare_api.sh but not by terraform module
+
+# export the variables for terraform
+#export TF_VAR_cloudflare_email="$CLOUDFLARE_EMAIL"
+#export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above
+
+# ============================================================================ #
+# Load External Envrc Files If Present
+# ============================================================================ #
+
+# XXX: safer to bring all these external .envrc inline if you're worried about changes
+# to it bypassing 'direnv allow' authorization
+load_if_exists(){
+ # first arg is a path to a .envrc
+ # all other args are passed to the sourcing of .envrc - used by .envrc-kubernetes
+ # to pass the context name 'docker-desktop' to switch to
+ local envrc="$1"
+ shift
+ if ! [[ "$envrc" =~ ^/ ]]; then
+ envrc="$srcdir/$envrc"
+ fi
+ if [ -f "$envrc" ]; then
+ # prevent looping on symlinks to this .envrc if given
+ if [ "$(readlink "$envrc")" = "$src" ]; then
+ return
+ fi
+ echo
+ echo "Loading $envrc"
+ # shellcheck disable=SC1090,SC1091
+ . "$envrc" "$@"
+ fi
+}
+
+# don't do this it may lead to an infinite loop if 'make link' symlinking ~/.envrc to this repo's .envrc
+# (which I do to keep Python virtual automatically loaded at all times because recent pip on Python refuses
+# to install to system Python)
+#load_if_exists ~/.envrc
+
+# ============================================================================ #
+# P y t h o n
+# ============================================================================ #
+
+ #.envrc-aws \
+ #.envrc-gcp \
+ #.envrc-terraform \
+# shellcheck disable=SC2043
+for envrc in \
+ .envrc-python \
+ ; do
+ load_if_exists "$envrc"
+done
+
+# ============================================================================ #
+# A W S
+# ============================================================================ #
+
+if [[ "$PWD" =~ /aws/ ]]; then
+ load_if_exists .envrc-aws
+fi
+
+# ============================================================================ #
+# G C P
+# ============================================================================ #
+
+if [[ "$PWD" =~ /gcp/ ]]; then
+ load_if_exists .envrc-gcp
+fi
+
+# ============================================================================ #
+# T e r r a f o r m
+# ============================================================================ #
+
+if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then
+ load_if_exists .envrc-terraform
+fi
+
+# ============================================================================ #
+# K u b e r n e t e s
+# ============================================================================ #
+
+if [ -f "$srcdir/.envrc-kubernetes" ]; then
+ load_if_exists .envrc-kubernetes docker-desktop
+fi
+
+# ============================================================================ #
+# . E n v
+# ============================================================================ #
+
+echo
+# read .env too
+#dotenv
+
+load_if_exists .envrc.local
diff --git a/.envrc-python b/.envrc-python
new file mode 100644
index 000000000..6bc2d65b1
--- /dev/null
+++ b/.envrc-python
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: Mon Feb 22 17:42:01 2021 +0000
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# P y t h o n D i r E n v
+# ============================================================================ #
+
+# .envrc to auto-load the virtualenv inside the 'venv' directory if present
+
+# https://direnv.net/man/direnv-stdlib.1.html
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default
+for venv in "$PWD/venv" "$HOME/venv"; do
+ if [ -f "$venv/bin/activate" ]; then
+ echo
+ echo "Virtualenv directory found in: $venv"
+ echo
+ echo "Activating Virtualenv inside the directory: $venv"
+
+ # shellcheck disable=SC1091
+ source "$venv/bin/activate"
+ break
+ fi
+done
+
+# read .env too
+#dotenv
diff --git a/.flake8 b/.flake8
index e31fb949f..de51ee708 100644
--- a/.flake8
+++ b/.flake8
@@ -1,5 +1,32 @@
+#
+# Author: Hari Sekhon
+# Date: Mon Oct 21 15:57:10 2019 +0100
+#
+# vim:ts=4:sts=4:sw=4:et
+#
+# https///github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# F l a k e 8 C o n f i g
+# ============================================================================ #
+
+# https://flake8.pycqa.org/en/latest/user/configuration.html
+
[flake8]
-ignore = E265,E402,F401
+
max-line-length = 120
+
+ignore = E265,
+ E402,
+ F401
+
exclude = test*/*
+
max-complexity = 10
diff --git a/.gitallowed b/.gitallowed
new file mode 100644
index 000000000..bb1dbfb76
--- /dev/null
+++ b/.gitallowed
@@ -0,0 +1,9 @@
+AKIAIOSFODNN7EXAMPLE
+
+wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+
+AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE
+
+ASIAIOSFODNN7EXAMPLE
+
+AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 000000000..ad861088b
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1,38 @@
+#
+# Author: Hari Sekhon
+# Date: 2021-11-09 15:14:59 +0000 (Tue, 09 Nov 2021)
+#
+# vim:ts=4:sts=4:sw=4:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
+
+# Good in theory, to alert on PR changes to these code paths, but for public repos which may be forked and run .github/workflows/fork-update.yaml, this will result in a lot of spam
+
+# Tips:
+#
+# * includes changes under .github/
+# dir/* only matches first level file changes but doesn't recurse
+# dir/ recurses
+#
+# - CODEOWNERS in base branch of PR determines review request
+# - paths are case sensitive
+# - last match wins, use * at top for overall owner then override with more specific teams
+
+#* @harisekhon # username or email address
+#* @myorg/platform-engineering # team based is the way to go - team must have Write access to the repo regardless of if individuals have access
+#* @myorg/devops
+#k8s @myorg/devops @myorg/sre-team
+#apps/ @myorg/developers
+#apps/dir2 # ignores dir2 as no owner/team specified on this line
+#src/ @myorg/developers
+#docs/ docs@example.com
+#.github/workflows @ci-cd-team
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index b951f3244..aa46b9847 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -1,7 +1,3 @@
-Please be as specific as possible when raising an issue.
+Please be specific about your issue and include debug output from running with `-v -v -v` or for shell scripts after setting `export DEBUG=1` in your shell.
-- what were you expecting
-- what was the result
-- what was the output - if running a CLI program please include the full debug output when running with program by using the `--debug` or `-v -v -v` switches for most tools, or setting `export DEBUG=1` environment variable, which also works with the shell scripts.
-
-You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-Tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-Tools) respectively.
+You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-tools) respectively.
diff --git a/.github/workflows/*_centos.yaml.disabled b/.github/workflows/*_centos.yaml.disabled
new file mode 100644
index 000000000..51d70819e
--- /dev/null
+++ b/.github/workflows/*_centos.yaml.disabled
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (CentOS)
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:centos
+ ghcr.io/harisekhon/pytools:centos
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-centos
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml
new file mode 100644
index 000000000..9bed93efb
--- /dev/null
+++ b/.github/workflows/alpine.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Alpine
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: alpine:latest
+ caches: apk pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml
new file mode 100644
index 000000000..f6c4f5ea9
--- /dev/null
+++ b/.github/workflows/alpine_3.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Alpine 3
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine_3.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: alpine:3
+ caches: apk pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/centos.yaml.disabled b/.github/workflows/centos.yaml.disabled
new file mode 100644
index 000000000..420b88392
--- /dev/null
+++ b/.github/workflows/centos.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: CentOS
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/centos.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: centos:latest
+ caches: yum pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/centos7.yaml.disabled b/.github/workflows/centos7.yaml.disabled
new file mode 100644
index 000000000..ec764d288
--- /dev/null
+++ b/.github/workflows/centos7.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: CentOS 7
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/centos7.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: centos:7
+ caches: yum pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/centos8.yaml.disabled b/.github/workflows/centos8.yaml.disabled
new file mode 100644
index 000000000..76c476664
--- /dev/null
+++ b/.github/workflows/centos8.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: CentOS 8
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/centos8.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: centos:8
+ caches: yum pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml
new file mode 100644
index 000000000..209b3cef1
--- /dev/null
+++ b/.github/workflows/checkov.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C h e c k o v G i t H u b W o r k f l o w
+# ============================================================================ #
+
+# Static analysis of Terraform code - publishes report to GitHub Security tab
+
+# https://github.com/bridgecrewio/checkov-action
+
+---
+name: Checkov
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+jobs:
+ checkov:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Checkov
+ uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml
new file mode 100644
index 000000000..168d4914c
--- /dev/null
+++ b/.github/workflows/codeowners.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C o d e O w n e r s
+# ============================================================================ #
+
+---
+name: CodeOwners
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - CODEOWNERS
+ - .github/CODEOWNERS
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - CODEOWNERS
+ - .github/CODEOWNERS
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ validate:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Validate CODEOWNERS
+ uses: HariSekhon/GitHub-Actions/.github/workflows/codeowners.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
new file mode 100644
index 000000000..5e74ab863
--- /dev/null
+++ b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,63 @@
+---
+name: "CodeQL"
+
+on:
+ push:
+ branches:
+ - master
+ pull_request:
+ # The branches below must be a subset of the branches above
+ branches:
+ - master
+ schedule:
+ - cron: '37 15 * * 4'
+
+jobs:
+ analyze:
+ name: Analyze
+ runs-on: ubuntu-latest
+ permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+ strategy:
+ fail-fast: false
+ matrix:
+ language:
+ - python
+ # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+ # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+
+ # Initializes the CodeQL tools for scanning.
+ - name: Initialize CodeQL
+ uses: github/codeql-action/init@v1
+ with:
+ languages: ${{ matrix.language }}
+ # If you wish to specify custom queries, you can do so here or in a config file.
+ # By default, queries listed here will override any specified in a config file.
+ # Prefix the list here with "+" to use these queries and those in the config file.
+ # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+ # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
+ # If this step fails, then you should remove it and run the build manually (see below)
+ - name: Autobuild
+ uses: github/codeql-action/autobuild@v1
+
+ # Command-line programs to run using the OS shell.
+ # https://git.io/JvXDl
+
+ # If the Autobuild fails above, remove it and uncomment the following three lines
+ # and modify them (or add more) to build your code if your project
+ # uses a compiled language
+
+ #- run: |
+ # make bootstrap
+ # make release
+
+ - name: Perform CodeQL Analysis
+ uses: github/codeql-action/analyze@v1
diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml
new file mode 100644
index 000000000..d84cb16b1
--- /dev/null
+++ b/.github/workflows/debian.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:latest
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml
new file mode 100644
index 000000000..e857f280f
--- /dev/null
+++ b/.github/workflows/debian_10.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 10
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_10.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:10
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml
new file mode 100644
index 000000000..ad765f1ad
--- /dev/null
+++ b/.github/workflows/debian_11.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 11
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_11.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:11
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_12.yaml b/.github/workflows/debian_12.yaml
new file mode 100644
index 000000000..dfbc4ec97
--- /dev/null
+++ b/.github/workflows/debian_12.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 12
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_12.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:12
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled
new file mode 100644
index 000000000..f3062b6f8
--- /dev/null
+++ b/.github/workflows/debian_6.yaml.disabled
@@ -0,0 +1,50 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 6
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:6
+ # causes nodejs errors
+ #caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled
new file mode 100644
index 000000000..1b0e00295
--- /dev/null
+++ b/.github/workflows/debian_7.yaml.disabled
@@ -0,0 +1,50 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 7
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:7
+ # causes nodejs errors
+ #caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_8.yaml.disabled b/.github/workflows/debian_8.yaml.disabled
new file mode 100644
index 000000000..b41ead7d3
--- /dev/null
+++ b/.github/workflows/debian_8.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 8
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_8.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:8
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_9.yaml.disabled b/.github/workflows/debian_9.yaml.disabled
new file mode 100644
index 000000000..451348091
--- /dev/null
+++ b/.github/workflows/debian_9.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 9
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_9.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:9
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml
new file mode 100644
index 000000000..7119d6852
--- /dev/null
+++ b/.github/workflows/docker_pytools_alpine.yaml
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (Alpine)
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:alpine
+ ghcr.io/harisekhon/pytools:alpine
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-alpine
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml
new file mode 100644
index 000000000..24f8f21dd
--- /dev/null
+++ b/.github/workflows/docker_pytools_debian.yaml
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (Debian)
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:debian
+ ghcr.io/harisekhon/pytools:debian
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-debian
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml
new file mode 100644
index 000000000..ad1a5ed8d
--- /dev/null
+++ b/.github/workflows/docker_pytools_fedora.yaml
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (Fedora)
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:fedora
+ ghcr.io/harisekhon/pytools:fedora
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-fedora
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml
new file mode 100644
index 000000000..884917919
--- /dev/null
+++ b/.github/workflows/docker_pytools_ubuntu.yaml
@@ -0,0 +1,126 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (Ubuntu)
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:latest
+ harisekhon/pytools:ubuntu
+ ghcr.io/harisekhon/pytools:latest
+ ghcr.io/harisekhon/pytools:ubuntu
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-ubuntu
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml
new file mode 100644
index 000000000..5a4f9a9b4
--- /dev/null
+++ b/.github/workflows/fedora.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Fedora
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/fedora.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: fedora
+ caches: yum pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml
new file mode 100644
index 000000000..6b228ca33
--- /dev/null
+++ b/.github/workflows/fork-sync.yaml
@@ -0,0 +1,48 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# F o r k S y n c
+# ============================================================================ #
+
+# For a fork of the original repo, activate to keep it up to date via straight GitHub sync to the default branch
+
+---
+name: Fork Sync
+
+on: # yamllint disable-line rule:truthy
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 */3 * * *'
+
+permissions:
+ contents: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: false
+
+jobs:
+ fork_sync:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == true
+ if: github.repository_owner != 'HariSekhon'
+ name: Fork Sync
+ uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml
new file mode 100644
index 000000000..c74fa2a0b
--- /dev/null
+++ b/.github/workflows/fork-update-pr.yaml
@@ -0,0 +1,51 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# F o r k U p d a t e P R
+# ============================================================================ #
+
+# For a fork of the original repo, activate to keep its branches up to date via Pull Requests
+#
+# To be used in conjunction with the adjacent fork-sync.yaml which keeps the default branch up to date
+
+---
+name: Fork Update PR
+
+on: # yamllint disable-line rule:truthy
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 10 * * 1'
+
+permissions:
+ contents: write
+ pull-requests: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: false
+
+jobs:
+ fork_update_pr:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == true
+ if: github.repository_owner != 'HariSekhon'
+ name: Fork Update PR
+ uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ghcr_python_ubuntu.yaml.disabled b/.github/workflows/ghcr_python_ubuntu.yaml.disabled
new file mode 100644
index 000000000..c4503bc13
--- /dev/null
+++ b/.github/workflows/ghcr_python_ubuntu.yaml.disabled
@@ -0,0 +1,36 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-02-09 18:07:10 +0000 (Wed, 09 Feb 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: GHCR Build (Ubuntu)
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ workflow_dispatch:
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build_ghcr.yaml@master
+ with:
+ image: pytools
+ tags: ubuntu latest
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-ubuntu
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml
new file mode 100644
index 000000000..1b0fa15b4
--- /dev/null
+++ b/.github/workflows/grype.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: 2023-05-13 01:07:56 +0100 (Sat, 13 May 2023)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# G r y p e
+# ============================================================================ #
+
+---
+name: Grype
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ Grype:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Grype
+ uses: HariSekhon/GitHub-Actions/.github/workflows/grype.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml
new file mode 100644
index 000000000..83fa5ef83
--- /dev/null
+++ b/.github/workflows/json.yaml
@@ -0,0 +1,58 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# J S O N
+# ============================================================================ #
+
+# Validate any JSON files found in the repo
+
+---
+name: JSON
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.json'
+ - .github/workflows/json.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.json'
+ - .github/workflows/json.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+jobs:
+ check_json:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Check JSON
+ uses: HariSekhon/GitHub-Actions/.github/workflows/json.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml
new file mode 100644
index 000000000..c389f5df0
--- /dev/null
+++ b/.github/workflows/kics.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-02-01 19:36:08 +0000 (Tue, 01 Feb 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# K i c s
+# ============================================================================ #
+
+---
+name: Kics
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ kics:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Kics
+ uses: HariSekhon/GitHub-Actions/.github/workflows/kics.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml
new file mode 100644
index 000000000..438dd184e
--- /dev/null
+++ b/.github/workflows/mac.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Mac
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ runs-on: macos-latest
+ caches: brew pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/mac_11.yaml b/.github/workflows/mac_11.yaml
new file mode 100644
index 000000000..cc392b105
--- /dev/null
+++ b/.github/workflows/mac_11.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Mac 11
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ runs-on: macos-11
+ caches: brew pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/mac_12.yaml b/.github/workflows/mac_12.yaml
new file mode 100644
index 000000000..0bcd6e6fd
--- /dev/null
+++ b/.github/workflows/mac_12.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Mac 12
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ runs-on: macos-12
+ caches: brew pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml
new file mode 100644
index 000000000..72e301ba0
--- /dev/null
+++ b/.github/workflows/markdown.yaml
@@ -0,0 +1,54 @@
+#
+# Author: Hari Sekhon
+# Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# M a r k D o w n
+# ============================================================================ #
+
+---
+name: Markdown
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.md'
+ - .mdlrc
+ - .mdl.rb
+ - .markdownlint.rb
+ - .github/workflows/markdown.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.md'
+ - .mdlrc
+ - .mdl.rb
+ - .markdownlint.rb
+ - .github/workflows/markdown.yaml
+ workflow_dispatch:
+
+permissions:
+ contents: read
+ pull-requests: read
+
+jobs:
+ Markdown:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Markdown
+ uses: HariSekhon/GitHub-Actions/.github/workflows/markdown.yaml@master
diff --git a/.github/workflows/pypy2.yaml.disabled b/.github/workflows/pypy2.yaml.disabled
new file mode 100644
index 000000000..01080bcc8
--- /dev/null
+++ b/.github/workflows/pypy2.yaml.disabled
@@ -0,0 +1,56 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: PyPy 2
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: PyPy2
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: pypy2
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/pypy3.yaml.disabled b/.github/workflows/pypy3.yaml.disabled
new file mode 100644
index 000000000..101b36751
--- /dev/null
+++ b/.github/workflows/pypy3.yaml.disabled
@@ -0,0 +1,56 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: PyPy 3
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: PyPy3
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: pypy3
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python2.7.yaml.disabled b/.github/workflows/python2.7.yaml.disabled
new file mode 100644
index 000000000..1813931e7
--- /dev/null
+++ b/.github/workflows/python2.7.yaml.disabled
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 2.7
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python2.7.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python2.7.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 2.7
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 2.7
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml
new file mode 100644
index 000000000..8ec3008b9
--- /dev/null
+++ b/.github/workflows/python3.10.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.10
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.10.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.10.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.10
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: "3.10"
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml
new file mode 100644
index 000000000..0f27b0076
--- /dev/null
+++ b/.github/workflows/python3.11.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.11
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.11.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.11.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.11
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: "3.11"
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.6.yaml.disabled b/.github/workflows/python3.6.yaml.disabled
new file mode 100644
index 000000000..91bd3e673
--- /dev/null
+++ b/.github/workflows/python3.6.yaml.disabled
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.6
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.6.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.6.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.6
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 3.6
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml
new file mode 100644
index 000000000..73a0045c7
--- /dev/null
+++ b/.github/workflows/python3.7.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.7
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.7.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.7.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.7
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 3.7
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml
new file mode 100644
index 000000000..abcbc4c68
--- /dev/null
+++ b/.github/workflows/python3.8.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.8
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.8.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.8.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.8
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 3.8
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml
new file mode 100644
index 000000000..1ce7eb238
--- /dev/null
+++ b/.github/workflows/python3.9.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.9
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.9.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.9.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.9
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 3.9
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml
new file mode 100644
index 000000000..c2f614222
--- /dev/null
+++ b/.github/workflows/semgrep-cloud.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# S e m g r e p C l o u d W o r k f l o w
+# ============================================================================ #
+
+# Logs results to https://semgrep.dev/
+
+---
+name: Semgrep Cloud
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ semgrep:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Semgrep Cloud
+ uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master
+ secrets:
+ SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }}
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml
new file mode 100644
index 000000000..04a3cf197
--- /dev/null
+++ b/.github/workflows/semgrep.yaml
@@ -0,0 +1,64 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# S e m g r e p G i t H u b W o r k f l o w
+# ============================================================================ #
+
+# Generates code scanning alerts in GitHub's Security tab -> Code scanning alerts
+
+# https://semgrep.dev/docs/semgrep-ci/sample-ci-configs/#github-actions
+
+---
+name: Semgrep
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ semgrep:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Semgrep GitHub Security Tab
+ uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml
new file mode 100644
index 000000000..246087bd4
--- /dev/null
+++ b/.github/workflows/shellcheck.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# S h e l l C h e c k
+# ============================================================================ #
+
+# Validate any shell scripts found in the repo
+
+---
+name: ShellCheck
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.sh'
+ - .github/workflows/shellcheck.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.sh'
+ - .github/workflows/shellcheck.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ shellcheck:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: ShellCheck
+ uses: HariSekhon/GitHub-Actions/.github/workflows/shellcheck.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml
new file mode 100644
index 000000000..95b2571ca
--- /dev/null
+++ b/.github/workflows/trivy.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# T r i v y
+# ============================================================================ #
+
+# Scan files in the local repo
+
+---
+name: Trivy
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ trivy:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Trivy
+ uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml
new file mode 100644
index 000000000..fe863f875
--- /dev/null
+++ b/.github/workflows/ubuntu.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:latest
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_14.04.yaml.disabled b/.github/workflows/ubuntu_14.04.yaml.disabled
new file mode 100644
index 000000000..1442d675d
--- /dev/null
+++ b/.github/workflows/ubuntu_14.04.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 14.04
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_14.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:14.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_16.04.yaml.disabled b/.github/workflows/ubuntu_16.04.yaml.disabled
new file mode 100644
index 000000000..a9655835d
--- /dev/null
+++ b/.github/workflows/ubuntu_16.04.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 16.04
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_16.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:16.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_18.04.yaml.disabled b/.github/workflows/ubuntu_18.04.yaml.disabled
new file mode 100644
index 000000000..4b510ba1e
--- /dev/null
+++ b/.github/workflows/ubuntu_18.04.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 18.04
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_18.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:18.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml
new file mode 100644
index 000000000..47d0512ac
--- /dev/null
+++ b/.github/workflows/ubuntu_20.04.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 20.04
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_20.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:20.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml
new file mode 100644
index 000000000..ba8a549b1
--- /dev/null
+++ b/.github/workflows/ubuntu_22.04.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 22.04
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_22.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:22.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml
new file mode 100644
index 000000000..0037e4ba3
--- /dev/null
+++ b/.github/workflows/ubuntu_github.yaml
@@ -0,0 +1,84 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: GitHub Actions Ubuntu
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_github.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml
new file mode 100644
index 000000000..ab6ab9638
--- /dev/null
+++ b/.github/workflows/validate.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# V a l i d a t i o n
+# ============================================================================ #
+
+# Run all custom validations against files in the repo
+
+---
+name: Validation
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ #schedule:
+ # - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ validate:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Validate
+ uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml
new file mode 100644
index 000000000..3f6265a6a
--- /dev/null
+++ b/.github/workflows/xml.yaml
@@ -0,0 +1,56 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# X M L
+# ============================================================================ #
+
+# Validate any XML files found in the repo
+
+---
+name: XML
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.xml'
+ - .github/workflows/xml.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.xml'
+ - .github/workflows/xml.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+jobs:
+ check_xml:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Check XML
+ uses: HariSekhon/GitHub-Actions/.github/workflows/xml.yaml@master
diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml
new file mode 100644
index 000000000..f2a4ca129
--- /dev/null
+++ b/.github/workflows/yaml.yaml
@@ -0,0 +1,64 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# Y A M L
+# ============================================================================ #
+
+# Validate any YAML files found in the repo
+
+---
+name: YAML
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.yml'
+ - '**/*.yaml'
+ - .github/workflows/yaml.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.yml'
+ - '**/*.yaml'
+ - .github/workflows/yaml.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ check_yaml:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Check YAML
+ uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.gitignore b/.gitignore
index f0ca6569a..3722dc711 100644
--- a/.gitignore
+++ b/.gitignore
@@ -48,7 +48,7 @@ dist/
downloads/
eggs/
.eggs/
-lib/
+#lib/ # breaks local lib/ change tracking
lib64/
parts/
sdist/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 000000000..625220b32
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,30 @@
+# vim:ts=2:sts=2:sw=2:et
+#
+# Author: Hari Sekhon
+# Date: Sun Feb 23 19:02:10 2020 +0000
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# G i t L a b C I
+# ============================================================================ #
+
+# https://docs.gitlab.com/ee/ci/yaml/README.html
+
+#include: '.gitlab/*.y*ml'
+
+image: ubuntu:18.04
+
+job:
+ before_script:
+ - setup/ci_bootstrap.sh
+ script:
+ - make init && make ci test
diff --git a/.gitmodules b/.gitmodules
index 37537e726..30fa60a67 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,8 +1,16 @@
[submodule "pylib"]
path = pylib
- url = https://github.com/harisekhon/pylib
+ url = https://github.com/HariSekhon/pylib
branch = master
[submodule "bash-tools"]
path = bash-tools
- url = https://github.com/harisekhon/bash-tools
+ url = https://github.com/HariSekhon/DevOps-Bash-tools
+ branch = master
+[submodule "sql"]
+ path = sql
+ url = https://github.com/HariSekhon/SQL-scripts
+ branch = master
+[submodule "templates"]
+ path = templates
+ url = https://github.com/HariSekhon/Templates
branch = master
diff --git a/.ipython-notebook-pyspark.00-pyspark-setup.py b/.ipython-notebook-pyspark.00-pyspark-setup.py
index 4dde897da..1de53b3fb 100644
--- a/.ipython-notebook-pyspark.00-pyspark-setup.py
+++ b/.ipython-notebook-pyspark.00-pyspark-setup.py
@@ -24,4 +24,4 @@
sys.path.insert(0, os.path.join(spark_home, 'python'))
for lib in glob.glob(os.path.join(spark_home, 'python/lib/py4j-*-src.zip')):
sys.path.insert(0, lib)
-execfile(os.path.join(spark_home, 'python/pyspark/shell.py'))
+execfile(os.path.join(spark_home, 'python/pyspark/shell.py')) # pylint: disable=undefined-variable
diff --git a/.mdl.rb b/.mdl.rb
new file mode 100644
index 000000000..f8f9b004c
--- /dev/null
+++ b/.mdl.rb
@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+# vim:ts=4:sts=4:sw=4:et:filetype=ruby
+#
+# Author: Hari Sekhon
+# Date: 2024-08-22 01:58:12 +0200 (Thu, 22 Aug 2024)
+#
+# https///github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+all
+#exclude_rule 'MD001'
+#exclude_rule 'MD003'
+#exclude_rule 'MD005'
+exclude_rule 'MD007' # leave 2 space indentation for lists, 3 space is ugly af
+#exclude_rule 'MD012'
+exclude_rule 'MD013' # long lines cannot be split if they are URLs
+#exclude_rule 'MD022'
+#exclude_rule 'MD025'
+#exclude_rule 'MD031'
+#exclude_rule 'MD032'
+exclude_rule 'MD033' # inline HTML is important for formatting
+exclude_rule 'MD036' # emphasis used instead of header for footer Ported from lines
+#exclude_rule 'MD039'
+#exclude_rule 'MD056'
diff --git a/.mdlrc b/.mdlrc
new file mode 100644
index 000000000..27e5b6895
--- /dev/null
+++ b/.mdlrc
@@ -0,0 +1,5 @@
+mdlrc_dir = File.expand_path('..', __FILE__)
+
+style_file = File.join(mdlrc_dir, '.mdl.rb')
+
+style style_file
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 000000000..787fb79ea
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,74 @@
+#
+# Author: Hari Sekhon
+# Date: 2024-08-08 17:34:56 +0300 (Thu, 08 Aug 2024)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https///github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# P r e - C o m m i t
+# ============================================================================ #
+
+---
+fail_fast: false
+#exclude: *.tmp$
+
+repos:
+
+ # will accept anything that 'git clone' understands
+ # this means you can set this to a local git repo to develop your own hook repos interactively
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v4.6.0
+ hooks:
+ - id: check-yaml
+ # Common errors
+ #- id: end-of-file-fixer # ruins .gitignore Icon\r
+ - id: trailing-whitespace
+ args: [--markdown-linebreak-ext=md]
+ # Git style
+ - id: check-added-large-files
+ - id: check-merge-conflict
+ - id: check-vcs-permalinks
+ #- id: forbid-new-submodules
+ # Cross platform
+ - id: check-case-conflict
+ - id: mixed-line-ending
+ args: [--fix=lf]
+ # Security
+ - id: detect-aws-credentials
+ args: ['--allow-missing-credentials']
+
+ # rewrites python files with useless changes like changing single quotes to double quotes
+ #- repo: https://github.com/psf/black
+ # rev: 24.8.0
+ # hooks:
+ # - id: black
+
+ # Git secrets Leaks
+ - repo: https://github.com/awslabs/git-secrets.git
+ # the release tags for 1.2.0, 1.2.1 and 1.3.0 are broken with this error:
+ #
+ # /Users/hari/.cache/pre-commit/repo......./.pre-commit-hooks.yaml is not a file
+ #
+ rev: 5357e18
+ hooks:
+ - id: git-secrets
+
+ - repo: https://github.com/markdownlint/markdownlint
+ rev: v0.12.0
+ hooks:
+ - id: markdownlint
+ name: Markdownlint
+ description: Run markdownlint on your Markdown files
+ entry: mdl
+ args: [-s, .mdl.rb]
+ language: ruby
+ files: \.(md|mdown|markdown)$
diff --git a/.pylintrc b/.pylintrc
index 65e97ab22..2006619da 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -1,378 +1,655 @@
-[MASTER]
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# P y L i n t C o n f i g
+# ============================================================================ #
+
+# pylint --generate-rcfile >> .pylintrc
+
+[MAIN]
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Clear in-memory caches upon conclusion of linting. Useful if running pylint
+# in a server-like mode.
+clear-cache-post-run=no
+
+# Load and enable all available extensions. Use --list-extensions to see a list
+# all available extensions.
+#enable-all-extensions=
+
+# In error mode, messages with a category besides ERROR or FATAL are
+# suppressed, and no reports are done by default. Error mode is compatible with
+# disabling specific errors.
+#errors-only=
+
+# Always return a 0 (non-error) status code, even if lint errors are found.
+# This is primarily useful in continuous integration scripts.
+#exit-zero=
-# Specify a configuration file.
-#rcfile=
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-allow-list=
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code. (This is an alternative name to extension-pkg-allow-list
+# for backward compatibility.)
+extension-pkg-whitelist=
+
+# Return non-zero exit code if any of these messages/categories are detected,
+# even if score is above --fail-under value. Syntax same as enable. Messages
+# specified are enabled, while categories only check already-enabled messages.
+fail-on=
+
+# Specify a score threshold under which the program will exit with error.
+fail-under=10
+
+# Interpret the stdin as a python script, whose filename needs to be passed as
+# the module_or_package argument.
+#from-stdin=
+
+# Files or directories to be skipped. They should be base names, not paths.
+ignore=CVS
+
+# Add files or directories matching the regular expressions patterns to the
+# ignore-list. The regex matches against paths and can be in Posix or Windows
+# format. Because '\\' represents the directory delimiter on Windows systems,
+# it can't be used as an escape character.
+ignore-paths=
+
+# Files or directories matching the regular expression patterns are skipped.
+# The regex matches against base names, not paths. The default value ignores
+# Emacs file locks
+ignore-patterns=^\.#
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis). It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
#init-hook=
-# Add files or directories to the blacklist. They should be base names, not
-# paths.
-ignore=CVS
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use, and will cap the count on Windows to
+# avoid hangs.
+jobs=1
-# Pickle collected data for later comparisons.
-persistent=yes
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
-# List of plugins (as comma separated values of python modules names) to load,
+# List of plugins (as comma separated values of python module names) to load,
# usually to register additional checkers.
load-plugins=
-# Use multiple processes to speed up Pylint.
-jobs=1
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Minimum Python version to use for version dependent checks. Will default to
+# the version used to run pylint.
+py-version=3.11
+
+# Discover python modules and packages in the file system subtree.
+recursive=no
+
+# Add paths to the list of the source roots. Supports globbing patterns. The
+# source root is an absolute path or a path relative to the current working
+# directory used to determine a package namespace for modules located under the
+# source root.
+source-roots=
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
unsafe-load-any-extension=no
-# A comma-separated list of package or module names from where C extensions may
-# be loaded. Extensions are loading into the active Python interpreter and may
-# run arbitrary code
-extension-pkg-whitelist=
+# In verbose mode, extra non-checker-related info will be displayed.
+#verbose=
-# Allow optimization of some AST trees. This will activate a peephole AST
-# optimizer, which will apply various small optimizations. For instance, it can
-# be used to obtain the result of joining multiple strings with the addition
-# operator. Joining a lot of strings can lead to a maximum recursion error in
-# Pylint and this flag can prevent that. It has one side effect, the resulting
-# AST will be different than the one from reality.
-optimize-ast=no
+[BASIC]
-[MESSAGES CONTROL]
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
-# Only show warnings with the listed confidence levels. Leave empty to show
-# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
-confidence=
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style. If left empty, argument names will be checked with the set
+# naming style.
+#argument-rgx=
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time. See also the "--disable" option for examples.
-#enable=
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifiers separated by comma (,) or put this
-# option multiple times (only on the command line, not in the configuration
-# file where it should appear only once).You can also use "--disable=all" to
-# disable everything first and then reenable specific checks. For example, if
-# you want to run only the similarities checker, you can use "--disable=all
-# --enable=similarities". If you want to run only the classes checker, but have
-# no Warning level messages displayed, use"--disable=all --enable=classes
-# --disable=W"
-disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,C0111
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style. If left empty, attribute names will be checked with the set naming
+# style.
+#attr-rgx=
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+ bar,
+ baz,
+ toto,
+ tutu,
+ tata
-[REPORTS]
+# Bad variable names regexes, separated by a comma. If names match any regex,
+# they will always be refused
+bad-names-rgxs=
-# Set the output format. Available formats are text, parseable, colorized, msvs
-# (visual studio) and html. You can also give a reporter class, eg
-# mypackage.mymodule.MyReporterClass.
-output-format=text
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]".
-files-output=no
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style. If left empty, class attribute names will be checked
+# with the set naming style.
+#class-attribute-rgx=
-# Tells whether to display a full report or only the messages
-reports=yes
+# Naming style matching correct class constant names.
+class-const-naming-style=UPPER_CASE
-# Python expression which should return a note less than 10 (10 is the highest
-# note). You have access to the variables errors warning, statement which
-# respectively contain the number of errors / warnings messages and the total
-# number of statements analyzed. This is used by the global evaluation report
-# (RP0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+# Regular expression matching correct class constant names. Overrides class-
+# const-naming-style. If left empty, class constant names will be checked with
+# the set naming style.
+#class-const-rgx=
-# Template used to display messages. This is a python new-style format string
-# used to format the message information. See doc for all details
-#msg-template=
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+# Regular expression matching correct class names. Overrides class-naming-
+# style. If left empty, class names will be checked with the set naming style.
+#class-rgx=
-[BASIC]
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style. If left empty, constant names will be checked with the set naming
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style. If left empty, function names will be checked with the set
+# naming style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+ j,
+ k,
+ ex,
+ Run,
+ _
+
+# Good variable names regexes, separated by a comma. If names match any regex,
+# they will always be accepted
+good-names-rgxs=
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style. If left empty, inline iteration names will be checked
+# with the set naming style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,filter,input
+# Regular expression matching correct method names. Overrides method-naming-
+# style. If left empty, method names will be checked with the set naming style.
+#method-rgx=
-# Good variable names which should always be accepted, separated by a comma
-good-names=i,j,k,ex,Run,_
+# Naming style matching correct module names.
+module-naming-style=snake_case
-# Bad variable names which should always be refused, separated by a comma
-bad-names=foo,bar,baz,toto,tutu,tata
+# Regular expression matching correct module names. Overrides module-naming-
+# style. If left empty, module names will be checked with the set naming style.
+#module-rgx=
# Colon-delimited sets of names that determine each other's naming style when
# the name regexes allow several styles.
name-group=
-# Include a hint for the correct naming format with invalid-name
-include-naming-hint=no
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
-# Regular expression matching correct function names
-function-rgx=[a-z_][a-z0-9_]{2,30}$
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
-# Naming hint for function names
-function-name-hint=[a-z_][a-z0-9_]{2,30}$
+# Regular expression matching correct type alias names. If left empty, type
+# alias names will be checked with the set naming style.
+#typealias-rgx=
-# Regular expression matching correct variable names
-variable-rgx=[a-z_][a-z0-9_]{2,30}$
+# Regular expression matching correct type variable names. If left empty, type
+# variable names will be checked with the set naming style.
+#typevar-rgx=
-# Naming hint for variable names
-variable-name-hint=[a-z_][a-z0-9_]{2,30}$
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
-# Regular expression matching correct constant names
-const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style. If left empty, variable names will be checked with the set
+# naming style.
+#variable-rgx=
-# Naming hint for constant names
-const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
-# Regular expression matching correct attribute names
-attr-rgx=[a-z_][a-z0-9_]{2,30}$
+[CLASSES]
-# Naming hint for attribute names
-attr-name-hint=[a-z_][a-z0-9_]{2,30}$
+# Warn about protected attribute access inside special methods
+check-protected-access-in-special-methods=no
-# Regular expression matching correct argument names
-argument-rgx=[a-z_][a-z0-9_]{2,30}$
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+ __new__,
+ setUp,
+ asyncSetUp,
+ __post_init__
-# Naming hint for argument names
-argument-name-hint=[a-z_][a-z0-9_]{2,30}$
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit
-# Regular expression matching correct class attribute names
-class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
-# Naming hint for class attribute names
-class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
-# Regular expression matching correct inline iteration names
-inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
-# Naming hint for inline iteration names
-inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
+[DESIGN]
-# Regular expression matching correct class names
-class-rgx=[A-Z_][a-zA-Z0-9]+$
+# List of regular expressions of class ancestor names to ignore when counting
+# public methods (see R0903)
+exclude-too-few-public-methods=
-# Naming hint for class names
-class-name-hint=[A-Z_][a-zA-Z0-9]+$
+# List of qualified class names to ignore when counting class parents (see
+# R0901)
+ignored-parents=
-# Regular expression matching correct module names
-module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+# Maximum number of arguments for function / method.
+max-args=5
-# Naming hint for module names
-module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
-# Regular expression matching correct method names
-method-rgx=[a-z_][a-z0-9_]{2,30}$
+# Maximum number of boolean expressions in an if statement (see R0916).
+max-bool-expr=5
-# Naming hint for method names
-method-name-hint=[a-z_][a-z0-9_]{2,30}$
+# Maximum number of branch for function / method body.
+max-branches=12
-# Regular expression which should only match function or class names that do
-# not require a docstring.
-no-docstring-rgx=^_
+# Maximum number of locals for function / method body.
+max-locals=15
-# Minimum line length for functions/classes that require docstrings, shorter
-# ones are exempt.
-docstring-min-length=-1
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
-[ELIF]
+# Maximum number of return / yield for function / method body.
+max-returns=6
-# Maximum number of nested blocks for function / method body
-max-nested-blocks=5
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when caught.
+overgeneral-exceptions=builtins.BaseException,builtins.Exception
[FORMAT]
-# Maximum number of characters on a single line.
-max-line-length=120
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=^\s*(# )??$
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string=' '
+
+# Maximum number of characters on a single line.
+max-line-length=120
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
# Allow the body of an if to be on the same line as the test if there is no
# else.
single-line-if-stmt=no
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=trailing-comma,dict-separator
-# Maximum number of lines in a module
-max-module-lines=1000
+[IMPORTS]
-# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
-# tab).
-indent-string=' '
+# List of modules that can be imported at any level, not just the top level
+# one.
+allow-any-import-level=
-# Number of spaces of indent required inside a hanging or continued line.
-indent-after-paren=4
+# Allow explicit reexports by alias from a package __init__.
+allow-reexport-from-package=no
-# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
-expected-line-ending-format=
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=
+
+# Output a graph (.gv or any supported image format) of external dependencies
+# to the given file (report RP0402 must not be disabled).
+ext-import-graph=
+
+# Output a graph (.gv or any supported image format) of all (i.e. internal and
+# external) dependencies to the given file (report RP0402 must not be
+# disabled).
+import-graph=
+
+# Output a graph (.gv or any supported image format) of internal dependencies
+# to the given file (report RP0402 must not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+# Couples of modules and preferred modules, separated by a comma.
+preferred-modules=
[LOGGING]
+# The type of string formatting that logging methods do. `old` means using %
+# formatting, `new` is for `{}` formatting.
+logging-format-style=old
+
# Logging modules to check that the string format arguments are in logging
-# function parameter format
+# function parameter format.
logging-modules=logging
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE,
+# UNDEFINED.
+confidence=HIGH,
+ CONTROL_FLOW,
+ INFERENCE,
+ INFERENCE_FAILURE,
+ UNDEFINED
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then re-enable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=raw-checker-failed,
+ bad-inline-option,
+ locally-disabled,
+ file-ignored,
+ suppressed-message,
+ useless-suppression,
+ deprecated-pragma,
+ use-symbolic-message-instead,
+ missing-class-docstring,
+ missing-function-docstring,
+ super-with-arguments,
+ consider-using-f-string
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[METHOD_ARGS]
+
+# List of qualified names (i.e., library.method) which require a timeout
+# parameter e.g. 'requests.api.get,requests.api.post'
+timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request
+
+
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
-notes=FIXME,XXX,TODO
+notes=FIXME,
+ XXX,
+ TODO
+# Regular expression of note tags to take in consideration.
+notes-rgx=
-[SIMILARITIES]
-# Minimum lines number of a similarity.
-min-similarity-lines=4
+[REFACTORING]
-# Ignore comments when computing similarities.
-ignore-comments=yes
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
-# Ignore docstrings when computing similarities.
-ignore-docstrings=yes
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit,argparse.parse_error
-# Ignore imports when computing similarities.
-ignore-imports=no
+[REPORTS]
-[SPELLING]
+# Python expression which should return a score less than or equal to 10. You
+# have access to the variables 'fatal', 'error', 'warning', 'refactor',
+# 'convention', and 'info' which contain the number of messages in each
+# category, as well as 'statement' which is the total number of statements
+# analyzed. This score is used by the global evaluation report (RP0004).
+evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))
-# Spelling dictionary name. Available dictionaries: none. To make it working
-# install python-enchant package.
-spelling-dict=
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+msg-template=
-# List of comma separated words that should not be checked.
-spelling-ignore-words=
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+#output-format=
-# A path to a file that contains private dictionary; one word per line.
-spelling-private-dict-file=
+# Tells whether to display a full report or only the messages.
+reports=no
-# Tells whether to store unknown words to indicated private dictionary in
-# --spelling-private-dict-file option instead of raising a message.
-spelling-store-unknown-words=no
+# Activate the evaluation score.
+score=yes
-[TYPECHECK]
+[SIMILARITIES]
-# Tells whether missing members accessed in mixin class should be ignored. A
-# mixin class is detected if its name ends with "mixin" (case insensitive).
-ignore-mixin-members=yes
+# Comments are removed from the similarity computation
+ignore-comments=yes
-# List of module names for which member attributes should not be checked
-# (useful for modules/projects where namespaces are manipulated during runtime
-# and thus existing member attributes cannot be deduced by static analysis. It
-# supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
+# Docstrings are removed from the similarity computation
+ignore-docstrings=yes
-# List of classes names for which member attributes should not be checked
-# (useful for classes with attributes dynamically set). This supports can work
-# with qualified names.
-ignored-classes=
+# Imports are removed from the similarity computation
+ignore-imports=yes
-# List of members which are set dynamically and missed by pylint inference
-# system, and so shouldn't trigger E1101 when accessed. Python regular
-# expressions are accepted.
-generated-members=
+# Signatures are removed from the similarity computation
+ignore-signatures=yes
+# Minimum lines number of a similarity.
+min-similarity-lines=4
-[VARIABLES]
-# Tells whether we should check for unused import in __init__ files.
-init-import=no
+[SPELLING]
-# A regular expression matching the name of dummy variables (i.e. expectedly
-# not used).
-dummy-variables-rgx=_$|dummy
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
-# List of additional names supposed to be defined in builtins. Remember that
-# you should avoid to define new builtins when possible.
-additional-builtins=
+# Spelling dictionary name. No available dictionaries : You need to install
+# both the python package and the system dependency for enchant to work..
+spelling-dict=
-# List of strings which can identify a callback function by name. A callback
-# name must start or end with one of those strings.
-callbacks=cb_,_cb
+# List of comma separated words that should be considered directives if they
+# appear at the beginning of a comment and should not be checked.
+spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
-[CLASSES]
+# A path to a file that contains the private dictionary; one word per line.
+spelling-private-dict-file=
-# List of method names used to declare (i.e. assign) instance attributes.
-defining-attr-methods=__init__,__new__,setUp
+# Tells whether to store unknown words to the private dictionary (see the
+# --spelling-private-dict-file option) instead of raising a message.
+spelling-store-unknown-words=no
-# List of valid names for the first argument in a class method.
-valid-classmethod-first-arg=cls
-# List of valid names for the first argument in a metaclass class method.
-valid-metaclass-classmethod-first-arg=mcs
+[STRING]
-# List of member names, which should be excluded from the protected access
-# warning.
-exclude-protected=_asdict,_fields,_replace,_source,_make
+# This flag controls whether inconsistent-quotes generates a warning when the
+# character used as a quote delimiter is used inconsistently within a module.
+check-quote-consistency=no
+# This flag controls whether the implicit-str-concat should generate a warning
+# on implicit string concatenation in sequences defined over several lines.
+check-str-concat-over-line-jumps=no
-[DESIGN]
-# Maximum number of arguments for function / method
-max-args=5
+[TYPECHECK]
-# Argument names that match this expression will be ignored. Default to name
-# with leading underscore
-ignored-argument-names=_.*
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
-# Maximum number of locals for function / method body
-max-locals=15
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
-# Maximum number of return / yield for function / method body
-max-returns=6
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
-# Maximum number of branch for function / method body
-max-branches=12
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
-# Maximum number of statements in function / method body
-max-statements=50
+# List of symbolic message names to ignore for Mixin members.
+ignored-checks-for-mixins=no-member,
+ not-async-context-manager,
+ not-context-manager,
+ attribute-defined-outside-init
-# Maximum number of parents for a class (see R0901).
-max-parents=7
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace
-# Maximum number of attributes for a class (see R0902).
-max-attributes=7
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
-# Minimum number of public methods for a class (see R0903).
-min-public-methods=2
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
-# Maximum number of public methods for a class (see R0904).
-max-public-methods=20
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
-# Maximum number of boolean expressions in a if statement
-max-bool-expr=5
+# Regex pattern to define which classes are considered mixins.
+mixin-class-rgx=.*[Mm]ixin
+# List of decorators that change the signature of a decorated function.
+signature-mutators=
-[IMPORTS]
-# Deprecated modules which should not be used, separated by a comma
-deprecated-modules=regsub,TERMIOS,Bastion,rexec
+[VARIABLES]
-# Create a graph of every (i.e. internal and external) dependencies in the
-# given file (report RP0402 must not be disabled)
-import-graph=
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
-# Create a graph of external dependencies in the given file (report RP0402 must
-# not be disabled)
-ext-import-graph=
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
-# Create a graph of internal dependencies in the given file (report RP0402 must
-# not be disabled)
-int-import-graph=
+# List of names allowed to shadow builtins
+allowed-redefined-builtins=
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+ _cb
-[EXCEPTIONS]
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
-# Exceptions that will emit a warning when being caught. Defaults to
-# "Exception"
-overgeneral-exceptions=Exception
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
new file mode 100644
index 000000000..59aa80836
--- /dev/null
+++ b/.semaphore/semaphore.yml
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-03-16 14:02:53 +0000 (Mon, 16 Mar 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# S e m a p h o r e C I
+# ============================================================================ #
+
+# https://docs.semaphoreci.com/reference/pipeline-yaml-reference/
+
+version: v1.0
+name: DevOps-Python-tools
+agent:
+ # https://docs.semaphoreci.com/reference/machine-types#linux
+ machine:
+ type: e1-standard-2
+ os_image: ubuntu2004
+execution_time_limit:
+ hours: 3
+blocks:
+ - name: Linux build
+ run:
+ when: "branch = 'master' AND change_in('/', {exclude: ['**/*.md']})"
+ #execution_time_limit:
+ # hours: 2
+ task:
+ #env_vars:
+ # $PATH selects /usr/bin/python and /usr/local/bin/pip which are mismatched versions of Python
+ #- name: PYTHON
+ # value: python3
+ #- name: PIP
+ # value: pip3
+ prologue:
+ commands:
+ - cache restore
+ # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server
+ #
+ # causes error:
+ #
+ # Not replacing deleted config file /etc/ssh/sshd_config
+ #
+ #- sudo rm -f /etc/ssh/sshd_config
+ - export DEBIAN_FRONTEND=noninteractive
+ - sudo -E apt-get update
+ - sudo -E apt-get upgrade -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confnew"
+ - sudo dpkg --configure -a --force-confmiss --force-confnew
+ #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections
+ #- sudo dpkg --configure -a --force-confdef --force-confold
+ #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"
+ - sudo apt-get install -y openssh-server
+ # each job is separate and could be run on a separate machine so all steps must be together
+ jobs:
+ - name: build
+ commands:
+ - checkout
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci
+ - make test
+ epilogue:
+ commands:
+ - cache store
+ - name: Mac build
+ run:
+ when: "branch = 'master'"
+ task:
+ # because otherwise on Mac it uses /usr/bin/python (2.7) but /usr/local/bin/pip (python 3.8)
+ #env_vars:
+ # to match /usr/local/bin/pip version from $PATH
+ #- name: PYTHON
+ # value: python3
+ # must be quoted to force string, otherwise pipeline fails to run with this parsing error:
+ # Error: [{"Type mismatch. Expected String but got Integer.", "#/blocks/1/task/env_vars/1/value"}]
+ #- name: DEBUG
+ # value: "1"
+ agent:
+ # https://docs.semaphoreci.com/reference/machine-types#macos
+ machine:
+ type: a1-standard-4
+ os_image: macos-xcode15
+ prologue:
+ commands:
+ - cache restore
+ # fix for:
+ # pip is configured with locations that require TLS/SSL, however the ssl module in Python is not available.
+ - brew install openssl
+ - brew reinstall python
+ - brew reinstall wget
+ # avoid Mac SSL errors:
+ #
+ # ERROR: Loading command: install (LoadError)
+ # dlopen(/Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle, 9): Library not loaded: /usr/local/opt/openssl/lib/libssl.1.0.0.dylib
+ # Referenced from: /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle
+ # Reason: image not found - /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle
+ # ERROR: While executing gem ... (NoMethodError)
+ # undefined method `invoke_with_build_args' for nil:NilClass#
+ #
+ - rbenv global system
+ # also considered this:
+ # - for version in $(rbenv versions | grep -v system | sed 's/^\*//'); do yes | rbenv uninstall "$version"; rbenv install "$version"; done
+ #
+ # fix for python vs pip version mismatch
+ - ln -svf -- /usr/local/bin/python3 /usr/local/bin/python
+ jobs:
+ - name: build
+ commands:
+ - checkout
+ - make init
+ - make ci
+ - make test
+ epilogue:
+ commands:
+ - cache store
diff --git a/.sonarcloud.properties b/.sonarcloud.properties
new file mode 100644
index 000000000..a4496869e
--- /dev/null
+++ b/.sonarcloud.properties
@@ -0,0 +1 @@
+sonar.host.url=https://sonarcloud.io
diff --git a/.travis.yml b/.travis.yml
index 0ef2d390c..47f2f80b5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,61 +3,137 @@
# Author: Hari Sekhon
# Date: 2014-11-29 01:02:47 +0000 (Sat, 29 Nov 2014)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+# ============================================================================ #
+# T r a v i s C I
+# ============================================================================ #
+
+# https://docs.travis-ci.com/user/customizing-the-build/
+
+---
version: ~> 1.0
+# ============================
# Mac Python 2.7 not available - see https://travis-ci.org/HariSekhon/pylib/jobs/584226228
#os:
# - linux
# - osx
+# =================================================
+# https://docs.travis-ci.com/user/languages/python/
+python:
+ - "2.7"
+ #- "3.4" supported by pip as of March 2019
+ - "3.5"
+ - "3.6"
+ - "3.7"
+ - "3.8"
+ - "pypy" # currently Python 2.7.13, PyPy 7.1.1
+ - "pypy3" # currently Python 3.6.1, PyPy 7.1.1-beta0
+
+# =============================================
+# https://docs.travis-ci.com/user/build-matrix/
+# https://docs.travis-ci.com/user/build-stages/matrix-expansion/
matrix:
+ fast_finish: true
include:
+ # numpy has gone 2.7+ only now, so had to drop Python 2.6 support
- os: linux
language: python
- python:
- # - "2.6"
- - "2.7"
- # MySQL in lib doesn't build from pip in Python 3
- # - "3.2"
- # - "3.3"
- # - "3.4"
- # - "3.5"
- # python-krbV fails to compile on PyPy
- # - "pypy"
- # - "pypy3"
- # workaround is to use generic and install to system python
+ python: "2.7"
+
- os: osx
- language: generic
+ language: generic # workaround since Mac doesn't have Python support yet, so install to system Python
+ # https://docs.travis-ci.com/user/reference/osx/
+ # macOS 10.15.7 - otherwise defaults to Mac macOS 10.13 with xcode9.4 otherwise - and HomeBrew update takes 50 minutes until the build times out :-/
+ osx_image: xcode12.2
+
+ - os: linux
+ language: python
+ python: "3.5"
+
+ - os: linux
+ language: python
+ python: "3.6"
+
+ - os: linux
+ language: python
+ python: "3.7"
-dist: trusty
+ - os: linux
+ language: python
+ python: "3.8"
-sudo: required
+ # python-krbV fails to compile on PyPy
+ #
+ # psutil doesn't build:
+ #
+ # RuntimeError: broken / incompatible Python implementation, see: https://github.com/giampaolo/psutil/issues/1659
+ #
+ #- os: linux
+ # language: python
+ # python: "pypy"
+ - os: linux
+ language: python
+ python: "pypy3"
+
+ # ===================================================================================================
+ # https://docs.travis-ci.com/user/multi-os/#allowing-failures-on-jobs-running-on-one-operating-system
+ allow_failures:
+ - python: "pypy"
+ - python: "pypy3"
+
+# =======================================
+# https://docs.travis-ci.com/user/docker/
+services:
+ - docker
+
+# ======================================================
+# https://docs.travis-ci.com/user/environment-variables/
env:
# - DEBUG=1
- DOCKER_COMPOSE_VERSION=1.16.1 PYTHONUNBUFFERED=1
+# ==============================================
+# https://docs.travis-ci.com/user/notifications/
notifications:
email: false
-branches:
- only:
- - master
+# =================================================================================
+# https://docs.travis-ci.com/user/customizing-the-build/#building-specific-branches
+# https://docs.travis-ci.com/user/conditional-builds-stages-jobs
+#branches:
+# only:
+# - master
-cache: pip
+# ========================================
+# https://docs.travis-ci.com/user/caching/
-services:
- - docker
+before_cache:
+ #- rm -f $HOME/.cache/pip/log/debug.log
+ # XXX: cache breaks pypy builds, so clear it
+ - rm -f $HOME/.cache/pip
+
+cache:
+ - pip
+ - directories:
+ - $HOME/.cache
+ - $HOME/.cpan
+ - $HOME/.cpanm
+ - $HOME/.gem
+
+# ==============================================
+# https://docs.travis-ci.com/user/job-lifecycle/
# avoid package checksum mismatches when installing packages
before_install:
diff --git a/.yamllint b/.yamllint
index b5e6f8448..e29153b7f 100644
--- a/.yamllint
+++ b/.yamllint
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2019-02-26 14:42:07 +0000 (Tue, 26 Feb 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
---
diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 000000000..d582bd112
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,88 @@
+// vim:ts=4:sts=4:sw=4:et:filetype=groovy:syntax=groovy
+//
+// Author: Hari Sekhon
+// Date: 2017-06-28 12:39:02 +0200 (Wed, 28 Jun 2017)
+//
+// https://github.com/HariSekhon/DevOps-Python-tools
+//
+// License: see accompanying Hari Sekhon LICENSE file
+//
+// If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+//
+// https://www.linkedin.com/in/HariSekhon
+//
+
+// ========================================================================== //
+// J e n k i n s P i p e l i n e
+// ========================================================================== //
+
+// Epic Jenkinsfile template:
+//
+// https://github.com/HariSekhon/Templates/blob/master/Jenkinsfile
+
+
+// Official Documentation:
+//
+// https://jenkins.io/doc/book/pipeline/syntax/
+//
+// https://www.jenkins.io/doc/pipeline/steps/
+//
+// https://www.jenkins.io/doc/pipeline/steps/workflow-basic-steps/
+
+
+pipeline {
+ // to run on Docker or Kubernetes, see the master Jenkinsfile template listed at the top
+ agent any
+
+ options {
+ timestamps()
+
+ timeout(time: 2, unit: 'HOURS')
+ }
+
+ triggers {
+ cron('H 10 * * 1-5')
+ pollSCM('H/2 * * * *')
+ }
+
+ stages {
+ stage ('Checkout') {
+ steps {
+ checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/HariSekhon/DevOps-Python-tools']]])
+ }
+ }
+
+ stage('Build') {
+ steps {
+ echo "Running ${env.JOB_NAME} Build ${env.BUILD_ID} on ${env.JENKINS_URL}"
+ echo 'Building...'
+ timeout(time: 10, unit: 'MINUTES') {
+ retry(3) {
+// sh 'apt update -q'
+// sh 'apt install -qy make'
+// sh 'make init'
+ sh """
+ setup/ci_bootstrap.sh &&
+ make init
+ """
+ }
+ }
+ timeout(time: 180, unit: 'MINUTES') {
+ sh 'make ci'
+ }
+ }
+ }
+
+ stage('Test') {
+ options {
+ retry(2)
+ }
+ steps {
+ echo 'Testing...'
+ timeout(time: 120, unit: 'MINUTES') {
+ sh 'make test'
+ }
+ }
+ }
+ }
+}
diff --git a/LICENSE b/LICENSE
index 03b527051..4860c1cf5 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,45 +1,7 @@
-=======================================
-HARI SEKHON LICENSE Revision 2013112300
-=======================================
+Copyright 2015 Hari Sekhon
-Copyright (c) 2006 onwards, Hari Sekhon
-All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-Redistribution and use in source and binary forms, with or without
-modification, is permitted provided that the following conditions are met:
-1. Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-3. All advertising materials mentioning features or use of this software
- must display the following acknowledgement:
- This product includes software developed by Hari Sekhon.
-4. Neither the name Hari Sekhon nor any affiliates may be used to endorse or
- promote products derived from this software without specific prior written
- permission.
-5. Modifications may be released to the public only with prior written permission
- from Hari Sekhon. Forking on GitHub is permitted for the purpose of creating
- patch pull requests back to the original repository.
-6. Private modifications may be made to suit requirements, but any modifications
- to this work, whether publicly disclosed or not, must be sent back to
- Hari Sekhon via GitHub (https://github.com/harisekhon/devops-python-tools) or
- LinkedIn (https://www.linkedin.com/in/harisekhon)
- and must come under this same license. Any such modifications may be
- reincorporated for the improvement of this software.
-7. This work may not be sold without prior written permission from Hari Sekhon
-8. This license may change at any time and the latest revision supersedes
- all prior revisions.
-9. Alternative licensing must be agreed in writing with Hari Sekhon prior
- to public availability.
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-THIS SOFTWARE IS PROVIDED BY Hari Sekhon ''AS IS'' AND ANY
-EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL Hari Sekhon OR ANY AFFILIATED BODY BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/Makefile b/Makefile
old mode 100755
new mode 100644
index f77f6f741..1d98ac524
--- a/Makefile
+++ b/Makefile
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# Travis has custom python install earlier in $PATH even in Perl builds so need to install PyPI modules to non-system python otherwise they're not found by programs.
@@ -15,20 +15,27 @@
# ===================
# bootstrap commands:
+# setup/bootstrap.sh
+#
+# OR
+#
# Alpine:
#
-# apk add --no-cache git $(MAKE) && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE)
-
+# apk add --no-cache git make && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make
+#
# Debian / Ubuntu:
#
-# apt-get update && apt-get install -y $(MAKE) git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE)
-
+# apt-get update && apt-get install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make
+#
# RHEL / CentOS:
#
-# yum install -y $(MAKE) git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE)
+# yum install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make
# ===================
+# would fail bootstrapping on Alpine
+#SHELL := /usr/bin/env bash
+
ifneq ("$(wildcard bash-tools/Makefile.in)", "")
include bash-tools/Makefile.in
endif
@@ -46,25 +53,58 @@ ifndef SKIP_PARQUET
endif
.PHONY: build
-build:
+build: init
@echo =========================
@echo DevOps Python Tools Build
@echo =========================
+ @$(MAKE) git-summary
+ @echo
+ # defer via external sub-call, otherwise will result in error like
+ # make: *** No rule to make target 'python-version', needed by 'build'. Stop.
+ @$(MAKE) python-version
- $(MAKE) init
if [ -z "$(CPANM)" ]; then make; exit $$?; fi
$(MAKE) system-packages-python
if type apk 2>/dev/null; then $(MAKE) apk-packages-extra; fi
if type apt-get 2>/dev/null; then $(MAKE) apt-packages-extra; fi
+
$(MAKE) python
.PHONY: init
init:
git submodule update --init --recursive
-.PHONY: python
-python:
+# Magic to build the dependencies for only given program(s)
+#
+# dependency of same % stem prefix checks for a matching .py file to consider it a valid target
+#
+# doesn't work
+#.PHONY: all
+#.PHONY: %.pyc
+# TODO: doesn't work, says nothing to be done even when .pyc isn't present, and allows make anonymize22.py which breaks
+#%.py: %.pyc
+# @$(MAKE) $@c
+%.pyc:: %.py
+ @# this utility script supports taking .pyc or .pyo names and still does the right thing
+ @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \
+ python -m py_compile $< && \
+ echo && \
+ echo Generated $@
+%.pyo:: %.py
+ @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \
+ python -O -m py_compile $< && \
+ echo && \
+ echo Generated $@
+
+.PHONY: pylib
+pylib:
+ @$(MAKE) python-version
cd pylib && $(MAKE)
+
+.PHONY: python
+python: pylib
+ # defer via external sub-call, otherwise will result in error like
+ # make: *** No rule to make target 'python-version', needed by 'build'. Stop.
@# don't pull parquet tools in to docker image by default, will bloat it
@# can fetch separately by running 'make parquet-tools' if you really want to
@if [ -f /.dockerenv -o -n "$(SKIP_PARQUET)" ]; then \
@@ -76,19 +116,20 @@ python:
fi
@# only install pip packages not installed via system packages
- @#$(SUDO_PIP) pip install --upgrade -r requirements.txt
- @#$(SUDO_PIP) pip install -r requirements.txt
- @PIP_OPTS="--ignore-installed" bash-tools/python_pip_install_if_absent.sh requirements.txt
+ @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt
+ @#$(SUDO_PIP) $(PIP) install -r requirements.txt
+ @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python/python_pip_install_if_absent.sh requirements.txt
- @# python-krbV dependency doesn't build on Mac any more and is unmaintained
+ @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3
@# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit
- @bash-tools/python_pip_install.sh snakebite[kerberos] || :
+ PIP=$(PIP) bash-tools/setup/python_install_snakebite.sh || :
# Python >= 3.4 - try but accept failure in case we're not on the right version of Python
- @if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi
+ @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3"; fi
+ PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3" || :
@# for impyla
- @#$(SUDO_PIP) pip install --upgrade setuptools || :
+ @#$(SUDO_PIP) $(PIP) install --upgrade setuptools || :
@#
@# snappy may fail to install on Mac not finding snappy-c.h - workaround:
@#
@@ -99,21 +140,21 @@ python:
@# /usr/local/include/snappy-c.h
@#
@# sudo su
- @# LD_RUN_PATH=/usr/local/include pip install snappy
+ @# LD_RUN_PATH=/usr/local/include $(PIP) install snappy
@#
- @#$(SUDO_PIP) pip install --upgrade -r requirements.txt
+ @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt
@# for ipython-notebook-pyspark.py
- @#$(SUDO_PIP) pip install jinja2
+ @#$(SUDO_PIP) PIP=$(PIP) install jinja2
@# HiveServer2
- @#$(SUDO_PIP) pip install pyhs2
+ @#$(SUDO_PIP) $(PIP) install pyhs2
@# Impala
- @#$(SUDO_PIP) pip install impyla
+ @#$(SUDO_PIP) $(PIP) install impyla
@# must downgrade happybase library to work on Python 2.6
- @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) pip install --upgrade "happybase==0.9"; fi
+ @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) $(PIP) install --upgrade "happybase==0.9"; fi
@# Python >= 2.7 - won't build on 2.6, handle separately and accept failure
- @bash-tools/python_pip_install.sh "ipython[notebook]" || :
+ @PIP=$(PIP) bash-tools/python/python_pip_install.sh "ipython[notebook]" || :
@echo
$(MAKE) pycompile
@echo
@@ -124,7 +165,7 @@ python:
.PHONY: parquet-tools
parquet-tools:
- @BIN='.' bash-tools/setup/install_parquet-tools.sh
+ @BIN='.' bash-tools/install/install_parquet-tools.sh
.PHONY: apk-packages-extra
apk-packages-extra:
@@ -134,7 +175,7 @@ apk-packages-extra:
.PHONY: apt-packages-extra
apt-packages-extra:
- if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk; fi
+ if [ -z "$(NOJAVA)" ]; then which java || bash-tools/packages/apt_install_packages.sh default-jdk; fi
# for validate_multimedia.py
# available in Alpine 2.6, 2.7 and 3.x
@@ -148,8 +189,8 @@ apk-packages-multimedia:
# Debian 9 Stretch onwards, not available in Debian 8 Jessie
.PHONY: apt-packages-multimedia
apt-packages-multimedia:
- $(SUDO) apt-get update
- $(SUDO) apt-get install -y --no-install-recommends ffmpeg
+ $(SUDO) apt-get update -o DPkg::Lock::Timeout=1200
+ $(SUDO) apt-get install -o DPkg::Lock::Timeout=1200 -y --no-install-recommends ffmpeg
# for validate_multimedia.py
.PHONY: yum-packages-multimedia
@@ -159,9 +200,10 @@ yum-packages-multimedia:
.PHONY: jython
jython:
- if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi
- if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi
- if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi
+ @#if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi
+ @#if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi
+ @#if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi
+ bash-tools/packages/install_packages.sh wget expect
sh jython_install.sh
.PHONY: test-lib
@@ -169,17 +211,13 @@ test-lib:
cd pylib && $(MAKE) test
.PHONY: test
-test: test-lib
+#test: test-lib
+test:
tests/all.sh
.PHONY: basic-test
basic-test: test-lib
- bash-tools/check_all.sh
-
-.PHONY: test2
-test2:
- cd pylib && $(MAKE) test2
- tests/all.sh
+ bash-tools/checks/check_all.sh
.PHONY: install
install: build
diff --git a/README.md b/README.md
index 27cd69ba9..5e1fe9bca 100644
--- a/README.md
+++ b/README.md
@@ -1,154 +1,385 @@
-Hari Sekhon - DevOps Python Tools
-=================================
-[](https://travis-ci.org/HariSekhon/DevOps-Python-tools)
-[](https://www.codacy.com/app/harisekhon/devops-python-tools)
-[](https://github.com/harisekhon/devops-python-tools/stargazers)
-[](https://github.com/harisekhon/devops-python-tools/network)
+# Hari Sekhon - DevOps Python Tools
+
+[](https://github.com/HariSekhon/DevOps-Python-tools/stargazers)
+[](https://github.com/HariSekhon/DevOps-Python-tools/network)
+[](https://github.com/boyter/scc/)
+[](https://github.com/boyter/scc/)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE)
+[](https://www.linkedin.com/in/HariSekhon/)
+[](https://github.com/HariSekhon/DevOps-Python-tools/commits/master)
+
+
+
+
+[](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard)
+[](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools)
+[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
+[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
+[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
+[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
+[](https://sonarcloud.io/summary/new_code?id=HariSekhon_DevOps-Python-tools)
+
+[](https://github.com/HariSekhon/DevOps-Python-tools)
+[](https://github.com/HariSekhon/DevOps-Python-tools)
+[](https://hub.docker.com/r/harisekhon/github/)
+[](https://github.com/HariSekhon/Dockerfiles)
+[](https://hub.docker.com/r/harisekhon/pytools)
+[](https://hub.docker.com/r/harisekhon/pytools/)
+[](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,Dockerfiles&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,HAProxy-configs&r=HariSekhon,SQL-scripts)
+[](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/STARCHARTS.md)
+
+
+[](https://harisekhon.github.io/CI-CD/)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.concourse.yml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.gocd.yml)
+[](https://github.com/HariSekhon/TeamCity-CI)
+
+[](https://circleci.com/gh/HariSekhon/DevOps-Python-tools)
+[](https://buildkite.com/hari-sekhon/devops-python-tools)
+[](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master)
+[](https://cloud.drone.io/HariSekhon/DevOps-Python-tools)
+[](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week)
+[](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools)
+[](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/shippable.yml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml)
+
+[](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master)
+[](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines)
+[](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/buildspec.yml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/cloudbuild.yaml)
+
+[](https://github.com/HariSekhon/DevOps-Python-tools)
+[](https://gitlab.com/HariSekhon/DevOps-Python-tools)
+[](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools)
+[](https://bitbucket.org/HariSekhon/DevOps-Python-tools)
+
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml)
+
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml)
+
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22GitHub+Actions+Ubuntu%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+22.04%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+11%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+12%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Fedora%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine+3%22)
+
+[](https://github.com/HariSekhon/DevOps-Python-tools)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.7%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.8%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.11%22)
+
+[git.io/pytools](https://git.io/pytools)
+
+## AWS, Docker, Spark, Hadoop, HBase, Hive, Impala, Python & Linux Tools
+
+DevOps, Cloud, Big Data, NoSQL, Python & Linux tools. All programs have `--help`.
Hari Sekhon
-Big Data Contractor, United Kingdom
+Cloud & Big Data Contractor, United Kingdom
-https://www.linkedin.com/in/harisekhon
+[](https://www.linkedin.com/in/HariSekhon/)
+
*(you're welcome to connect with me on LinkedIn)*
-##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. #####
+**Make sure you run `make update` if updating and not just `git pull` as you will often need the latest library submodule and possibly new upstream libraries**
-### Quick Start ###
+## Quick Start
-#### Ready to run Docker image #####
+### Ready to run Docker image
All programs and their pre-compiled dependencies can be found ready to run on [DockerHub](https://hub.docker.com/r/harisekhon/pytools/).
List all programs:
-```
+
+```shell
docker run harisekhon/pytools
```
+
Run any given program:
-```
+
+```shell
docker run harisekhon/pytools
```
-#### Automated Build from source #####
+### Automated Build from source
+
+installs git, make, pulls the repo and build the dependencies:
+```shell
+curl -L https://git.io/python-bootstrap | sh
```
-git clone https://github.com/harisekhon/devops-python-tools pytools
+
+or manually:
+
+```shell
+git clone https://github.com/HariSekhon/DevOps-Python-tools pytools
cd pytools
make
```
-Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/devops-python-tools#detailed-build-instructions) further down for more information.
+To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension
+instead of `.py`:
+
+```shell
+make anonymize.pyc
+```
+
+Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/DevOps-Python-tools#detailed-build-instructions) further down for more information.
-Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/harisekhon/devops-python-tools#jython-for-hadoop-utils) for details.
+Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/HariSekhon/DevOps-Python-tools#jython-for-hadoop-utils) for details.
-### Usage ###
+### Usage
-All programs come with a ```--help``` switch which includes a program description and the list of command line options.
+All programs come with a `--help` switch which includes a program description and the list of command line options.
-Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. ```$PASSWORD```, ```$TRAVIS_TOKEN```. These are indicated in the ```--help``` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. ```$AMBARI_HOST```, ```$HBASE_HOST``` take priority over ```$HOST```.
+Environment variables are supported for convenience and also to hide credentials from being exposed in the process list
+eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and
+often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`.
-### DevOps Python Tools ###
+### DevOps Python Tools - Inventory
- Linux:
- - ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists)
- - anonymizes:
+ - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing
+ - lists)
+ - anonymizations include these and more:
- hostnames / domains / FQDNs
- email addresses
- IP + MAC addresses
+ - AWS Access Keys, Secret Keys, ARNs, STS tokens
- Kerberos principals
- LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...)
- Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings
- - ```anonymize_custom.conf``` - put regex of your Name/Company/Project/Database/Tables to anonymize to ``````
- - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````)
- - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications
- - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters
- - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files
- - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename
- - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo)
-- [Hadoop](http://hadoop.apache.org/) & NoSQL:
- - [Spark](https://spark.apache.org/) & Data Format Converters:
- - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter
- - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter
- - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas
- - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas
- - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter
- - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter
- - ```json_to_xml.py``` - JSON to XML converter
- - ```xml_to_json.py``` - XML to JSON converter
- - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json'
- - see also ```validate_*.py``` further down for all these formats and more
+ - `anonymize_custom.conf` - put regex of your Name/Company/Project/Database/Tables to anonymize to ``
+ - placeholder tokens indicate what was stripped out (eg. ``, ``, ``)
+ - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes
+ communicating with each other to compare configs and log communications
+ - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support
+ teams can differentiate hosts in clusters
+ - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel
+ before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation
+ important for anonymization rules, as well as maintaining file content order. On servers this parallelization can
+ result in a 30x speed up for large log files
+ - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file
+ basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename
+ - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments,
+ useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc).
+ Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See
+ further down for more details and sub-programs that simplify usage for many of the most common cluster technologies
+ - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in
+ your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo)
+- [Amazon Web Services](https://aws.amazon.com/):
+ - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally
+ filters for active keys and older than N days (for key rotation governance)
+ - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have
+ never been used (these should generally be removed/disabled). Optionally filters for only active keys
+ - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys.
+ Optionally filters for users not used in the last N days to find old accounts to remove
+ - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for
+ users with passwords not used in the last N days
+- [Google Cloud Platform](https://cloud.google.com/):
+ - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python:
+ - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to
+ [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is
+ triggered by [Cloud Scheduler](https://cloud.google.com/scheduler)
+ - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and
+ [Cloud Scheduler](https://cloud.google.com/scheduler) jobs
+ - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP
+ address - use this to test your VPC connector public routing, comparison with firewall rules etc.
+ - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its
+ accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via
+ VPC connector routing
+ - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with
+ their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days
+- [Docker](https://www.docker.com/):
+ - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a
+ docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very
+ useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only
+ the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests
+ across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh`
+ - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official
+ `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned
+ to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy
+ shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg.
+ `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all
+ docker images eg. `docker_pull_all_images_all_tags.sh`
+ - `dockerfiles_check_git*.py` - check Git tags & branches align with the containing Dockerfile's `ARG *_VERSION`
+- [Spark](https://spark.apache.org/) & Data Format Converters:
+ - `spark_avro_to_parquet.py` - PySpark Avro => Parquet converter
+ - `spark_parquet_to_avro.py` - PySpark Parquet => Avro converter
+ - `spark_csv_to_avro.py` - PySpark CSV => Avro converter, supports both inferred and explicit schemas
+ - `spark_csv_to_parquet.py` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas
+ - `spark_json_to_avro.py` - PySpark JSON => Avro converter
+ - `spark_json_to_parquet.py` - PySpark JSON => Parquet converter
+ - `xml_to_json.py` - XML to JSON converter
+ - `json_to_xml.py` - JSON to XML converter
+ - `json_to_yaml.py` - JSON to YAML converter
+ - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for
+ pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and
+ [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json /
+ directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard
+ output for convenient command line chaining and redirection, optionally continues on error, collects broken records
+ to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not
+ technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json'
+ - `yaml_to_json.py` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON)
+ - see also `validate_*.py` further down for all these formats and more
+- [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL:
- [Ambari](https://hortonworks.com/apache/ambari/):
- - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API
+ - `ambari_blueprints.py` - Blueprint cluster templating and deployment tool using Ambari API
- list blueprints
- fetch all blueprints or a specific blueprint to local json files
- blueprint an existing cluster
- create a new cluster using a blueprint
- - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control
+ - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for
+ proper revision control
- optionally strips out the excessive and overly specific configs to create generic more reusable templates
- - see the ```ambari_blueprints/``` directory for a variety of Ambari blueprint templates generated by and deployable using this tool
- - ```ambari_ams_*.sh``` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts
- - ```ambari_cancel_all_requests.sh``` - cancel all ongoing operations using the Ambari API
- - ```ambari_trigger_service_checks.py``` - trigger service checks using the Ambari API
+ - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable
+ using this tool
+ - `ambari_ams_*.sh` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts
+ - `ambari_cancel_all_requests.sh` - cancel all ongoing operations using the Ambari API
+ - `ambari_trigger_service_checks.py` - trigger service checks using the Ambari API
- [Hadoop](http://hadoop.apache.org/) HDFS:
- - ```hadoop_hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports.
- - ```hadoop_hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum)
- - ```hadoop_hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files
+ - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to
+ replication factor 3 to avoid missing block alerts during datanode maintenance windows
+ - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file
+ or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data
+ layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports.
+ - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster
+ than doing `hdfs dfs -cat | md5sum`)
+ - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree
+ showing block size, replication factor, underfilled blocks and small files
- [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/):
- - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output
+ - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format
+ to standard output
+
+ The following programs can all optionally filter by database / table name regex:
+
+ - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala
+ table
+ - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation
+ between cluster migrations
+ - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for
+ finding unusually wide tables
+ - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts.
+ Useful for finding unusually big tables
+ - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where
+ any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or
+ subtle ETL bugs
+ - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs.
+ Useful for catching data quality problems or subtle ETL bugs
+ - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain
+ NULLs. Useful for catching data quality problems or subtle ETL bugs
+ - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL
+ property from describe table
+ - `hive_tables_locations.py` / `impala_tables_locations.py` - outputs for each table its data location
- [HBase](https://hbase.apache.org/):
- - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written.
- - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables
- - ```hbase_table_region_row_distribution.py``` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions
- - ```hbase_table_row_key_distribution.py``` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix
- - ```hbase_compact_tables.py``` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables.
- - ```hbase_flush_tables.py``` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables.
- - ```hbase_regions_by_*size.py``` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending
- - ```hbase_region_requests.py``` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting
- - ```hbase_regionserver_requests.py``` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting
- - ```hbase_regions_least_used.py``` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold)
+ - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table,
+ with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI
+ tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written.
+ - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting
+ tables
+ - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table,
+ giving per region row counts and % of total rows for the table as well as median and quartile row counts per
+ regions
+ - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in
+ an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row
+ counts per prefix
+ - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating
+ on all tables or takes an optional regex and compacts only matching tables.
+ - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an
+ optional regex and flushes only matching tables.
+ - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or
+ memStoreSize, ascending or descending
+ - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average
+ since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes /
+ total requests per second. Useful for watching more granular region stats to detect region hotspotting
+ - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or
+ average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read,
+ write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular
+ RegionServer stats to detect RegionServer hotspotting
+ - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received
+ the least requests (requests below a given threshold)
- [OpenTSDB](http://opentsdb.net/):
- - ```opentsdb_import_metric_distribution.py``` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting
- - ```opentsdb_list_metrics*.sh``` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending
+ - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew
+ and help avoid HBase region hotspotting
+ - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase
+ tables with optionally their created date, sorted ascending
- [Pig](https://pig.apache.org/)
- - ```pig-text-to-elasticsearch.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch)
- - ```pig-text-to-solr.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud)
- - ```pig_udfs.jy``` - Pig Jython UDFs for Hadoop
- - ```ipython-notebook-pyspark.py``` - per-user authenticated IPython Notebook + PySpark integration to allow each user to auto-create their own password protected IPython Notebook running Spark
- - ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch)
- - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required
- - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA
- - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA
- - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA
- - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing)
- - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing)
- - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node
- - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node
- - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore
- - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator
- - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server
- - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server
- - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node
- - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node
- - see also: [Advanced HAProxy configurations](https://github.com/harisekhon/haproxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins)
-- [Docker](https://www.docker.com/):
- - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh```
- - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (official `docker search` is limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh```
- - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION```
+ - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to
+ [Elasticsearch](https://www.elastic.co/products/elasticsearch)
+ - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to
+ [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud)
+ - `pig_udfs.jy` - Pig Jython UDFs for Hadoop
+- `find_active_server.py` - returns first available healthy server or active master in high availability deployments,
+ useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex
+ content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for
+ which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you
+ want to query cluster wide information available from any online peer (eg. Elasticsearch)
+ - The following are simplified specialisations of the above program, just pass host arguments, all the details have
+ been baked in, no switches required
+ - `find_active_hadoop_namenode.py` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA
+ - `find_active_hadoop_resource_manager.py` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA
+ - `find_active_hbase_master.py` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA
+ - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run
+ multiple of these for load balancing)
+ - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server
+ (run multiple of these for load balancing)
+ - `find_active_apache_drill.py` - returns first available [Apache Drill](https://drill.apache.org/) node
+ - `find_active_cassandra.py` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node
+ - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad,
+ Catalog or Statestore
+ - `find_active_presto_coordinator.py` - returns first available [Presto](https://prestodb.io/) Coordinator
+ - `find_active_kubernetes_api.py` - returns first available [Kubernetes](https://kubernetes.io/) API server
+ - `find_active_oozie.py` - returns first active [Oozie](http://oozie.apache.org/) server
+ - `find_active_solrcloud.py` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node
+ - `find_active_elasticsearch.py` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node
+ - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the
+ [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins)
- [Travis CI](https://travis-ci.org/):
- - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red
- - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI
+ - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo -
+ useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red
+ - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks
+ session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot
+ debug launcher for Travis CI
+- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as
+ Chrome and Firefox to fetch a given URL and content/regex match the result
- Data Validation (useful in CI):
- - ```validate_*.py``` - validate files, directory trees and/or standard input streams
+ - `validate_*.py` - validate files, directory trees and/or standard input streams
- supports the following file formats:
- Avro
- CSV
@@ -158,40 +389,44 @@ Environment variables are supported for convenience and also to hide credentials
- Parquet
- XML
- YAML
- - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`)
- - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & Linux technologies
+ - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`,
+ `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`)
+ - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for
+ things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images
+ [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data &
+ Linux technologies
### Detailed Build Instructions
-##### Python VirtualEnv localized installs
-
-The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages instead of installing from PyPI then follow the Manual Build section below.
+#### Python VirtualEnv localized installs
+The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it
+detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages
+instead of installing from PyPI then follow the Manual Build section below.
-#### Manual Setup
+### Manual Setup
Enter the pytools directory and run git submodule init and git submodule update to fetch my library repo:
-```
-git clone https://github.com/harisekhon/devops-python-tools pytools
+```shell
+git clone https://github.com/HariSekhon/DevOps-Python-tools pytools
cd pytools
git submodule init
git submodule update
sudo pip install -r requirements.txt
```
-
-#### Offline Setup
+### Offline Setup
Download the DevOps Python Tools and Pylib git repos as zip files:
-https://github.com/HariSekhon/devops-python-tools/archive/master.zip
+
-https://github.com/HariSekhon/pylib/archive/master.zip
+
-Unzip both and move Pylib to the ```pylib``` folder under DevOps Python Tools.
+Unzip both and move Pylib to the `pylib` folder under DevOps Python Tools.
-```
+```shell
unzip devops-python-tools-master.zip
unzip pylib-master.zip
@@ -200,153 +435,246 @@ mv -v pylib-master pylib
mv -vf pylib pytools/
```
-Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions).
+Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal
+mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions).
All PyPI modules are listed in the `requirements.txt` and `pylib/requirements.txt` files.
Internal Mirror example ([JFrog Artifactory](https://jfrog.com/artifactory/) or similar):
-```
+```shell
sudo pip install --index https://host.domain.com/api/pypi/repo/simple --trusted host.domain.com -r requirements.txt
```
Proxy example:
-```
+```shell
sudo pip install --proxy hari:mypassword@proxy-host:8080 -r requirements.txt
```
-##### Mac OS X
+#### Mac OS X
-The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing `git` is enough to trigger Xcode install).
+The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing
+`git` is enough to trigger Xcode install).
-I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X):
+I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for
+development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X):
-```
-bash-tools/setup/install_homebrew.sh
+```shell
+bash-tools/install/install_homebrew.sh
```
-```
+```shell
brew install openssl wget
```
If failing to build an OpenSSL lib dependency, just prefix the build command like so:
-```
+```shell
sudo OPENSSL_INCLUDE=/usr/local/opt/openssl/include OPENSSL_LIB=/usr/local/opt/openssl/lib ...
```
-You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is caused by pip 10 vs pip 9 and downgrading will work around it:
+You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is
+caused by pip 10 vs pip 9 and downgrading will work around it:
-```
+```shell
sudo pip install --upgrade pip==9.0.1
make
sudo pip install --upgrade pip
make
```
-### Jython for Hadoop Utils ###
+### Jython for Hadoop Utils
The 3 Hadoop utility programs listed below require Jython (as well as Hadoop to be installed and correctly configured)
-```
-hadoop_hdfs_time_block_reads.jy
-hadoop_hdfs_files_native_checksums.jy
-hadoop_hdfs_files_stats.jy
+```shell
+hdfs_time_block_reads.jy
+hdfs_files_native_checksums.jy
+hdfs_files_stats.jy
```
Run like so:
-```
-jython -J-cp $(hadoop classpath) hadoop_hdfs_time_block_reads.jy --help
+
+```shell
+jython -J-cp $(hadoop classpath) hdfs_time_block_reads.jy --help
```
-The ```-J-cp $(hadoop classpath) ``` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs.
+The `-J-cp $(hadoop classpath)` part dynamically inserts the current Hadoop java classpath required to use the Hadoop
+APIs.
See below for procedure to install Jython if you don't already have it.
-##### Automated Jython Install
+#### Automated Jython Install
This will download and install jython to /opt/jython-2.7.0:
-```
+```shell
make jython
```
-##### Manual Jython Install
+#### Manual Jython Install
-Jython is a simple download and unpack and can be fetched from http://www.jython.org/downloads.html
+Jython is a simple download and unpack and can be fetched from
Then add the Jython install bin directory to the $PATH or specify the full path to the `jython` binary, eg:
+```shell
+/opt/jython-2.7.0/bin/jython hdfs_time_block_reads.jy ...
```
-/opt/jython-2.7.0/bin/jython hadoop_hdfs_time_block_reads.jy ...
-```
-
-#### Configuration for Strict Domain / FQDN validation ####
+### Configuration for Strict Domain / FQDN validation
-Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/harisekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments).
+Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my
+[PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`,
+`.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal
+environments).
-#### Python SSL certificate verification problems
+### Python SSL certificate verification problems
If you end up with an error like:
-```
+
+```shell
./dockerhub_show_tags.py centos ubuntu
[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:765)
```
-It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One quick fix is to do the following:
-```
+
+It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One
+quick fix is to do the following:
+
+```shell
sudo pip uninstall -y certifi &&
sudo pip install certifi==2015.04.28
```
-### Updating ###
+### Updating
-Run ```make update```. This will git pull and then git submodule update which is necessary to pick up corresponding library updates.
+Run:
+
+```shell
+make update
+```
-If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run ```make update-no-recompile``` (will miss new library dependencies - do full ```make update``` if you encounter issues).
+This will git pull and then git submodule update which is necessary to pick up corresponding library updates.
+
+If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each
+time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter
+issues).
### Testing
[Continuous Integration](https://travis-ci.org/HariSekhon/devops-python-tools) is run on this repo with tests for success and failure scenarios:
-- unit tests for the custom supporting [python library](https://github.com/harisekhon/pylib)
+
+- unit tests for the custom supporting [python library](https://github.com/HariSekhon/pylib)
- integration tests of the top level programs using the libraries for things like option parsing
-- [functional tests](https://github.com/HariSekhon/devops-python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/)
+- [functional tests](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/)
To trigger all tests run:
-```
+```shell
make test
```
-which will start with the underlying libraries, then move on to top level integration tests and functional tests using docker containers if docker is available.
+which will start with the underlying libraries, then move on to top level integration tests and functional tests using
+docker containers if docker is available.
-### Contributions ###
+### Contributions
Patches, improvements and even general feedback are welcome in the form of GitHub pull requests and issue tickets.
-### See Also ###
+You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another
+Hortonworks guy Jonas Straub:
+
+
+
+## Star History
+
+[](https://star-history.com/#HariSekhon/DevOps-Python-tools&Date)
+
+[git.io/python-tools](https://git.io/python-tools)
+
+[git.io/pytools](https://git.io/pytools)
+
+## More Core Repos
+
+
+
+### Knowledge
+
+[](https://github.com/HariSekhon/Knowledge-Base)
+[](https://github.com/HariSekhon/Diagrams-as-Code)
+
+
+
+### DevOps Code
+
+[](https://github.com/HariSekhon/DevOps-Bash-tools)
+[](https://github.com/HariSekhon/DevOps-Python-tools)
+[](https://github.com/HariSekhon/DevOps-Perl-tools)
+[](https://github.com/HariSekhon/DevOps-Golang-tools)
+
+
+
+### Containerization
+
+[](https://github.com/HariSekhon/Kubernetes-configs)
+[](https://github.com/HariSekhon/Dockerfiles)
+
+### CI/CD
+
+[](https://github.com/HariSekhon/GitHub-Actions)
+[](https://github.com/HariSekhon/Jenkins)
+
+### Databases - DBA - SQL
+
+[](https://github.com/HariSekhon/SQL-scripts)
+
+### DevOps Reloaded
-* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (Hive, Impala, MySQL, PostgreSQL, Cassandra CQL, Apache Drill, Couchbase N1QL, Microsoft SQL Server, Oracle, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux...
+[](https://github.com/HariSekhon/HAProxy-configs)
+[](https://github.com/HariSekhon/Terraform)
+[](https://github.com/HariSekhon/Packer)
+[](https://github.com/HariSekhon/Ansible)
+[](https://github.com/HariSekhon/Environments)
-* [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Hadoop, Docker, Kafka, Elasticsearch, RabbitMQ, Redis, HBase, Solr, Cassandra, ZooKeeper, HDFS, Yarn, Hive, Presto, Drill, Impala, Consul, Spark, Jenkins, Travis CI, Git, MySQL, Linux, DNS, Whois, SSL Certs, Yum Security Updates, Kubernetes, Mesos, Riak, MongoDB, Memcached, Couchbase, CouchDB, Neo4j, Ambari, Cloudera, Hortonworks, MapR etc.
+### Monitoring
-* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 80+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies
+[](https://github.com/HariSekhon/Nagios-Plugins)
+[](https://github.com/HariSekhon/Nagios-Plugin-Kafka)
+[](https://github.com/HariSekhon/Prometheus)
-* [HAProxy-configs](https://github.com/harisekhon/haproxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc.
+### Templates
-* [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) - 50+ DockerHub public images for Docker & Kubernetes - Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Mesos, Consul, Riak, OpenTSDB, Jython, Advanced Nagios Plugins & DevOps Tools repos on Alpine, CentOS, Debian, Fedora, Ubuntu, Superset, H2O, Serf, Alluxio / Tachyon, FakeS3
+[](https://github.com/HariSekhon/Templates)
+[](https://github.com/HariSekhon/Template-repo)
-* [PyLib](https://github.com/harisekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule
+### Desktop
-* [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library
+[](https://github.com/HariSekhon/TamperMonkey)
+[](https://github.com/HariSekhon/Hammerspoon)
+[](https://github.com/HariSekhon/MPV-Scripts)
-* [Spark Apps eg. Spark => Elasticsearch](https://github.com/harisekhon/spark-to-elasticsearch) - Scala application to index from Spark to Elasticsearch. Used to index data in Hadoop clusters or local data via Spark standalone. This started as a Scala Spark port of ```pig-text-to-elasticsearch.pig``` from this repo.
+### Spotify
-You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub:
+[](https://github.com/HariSekhon/Spotify-tools)
+[](https://github.com/HariSekhon/Spotify-playlists)
-* https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb
+The rest of my original source repos are
+[here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers).
-### Stargazers over time
+Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/)
+and can be re-generated using the my [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) repo.
-[](https://starchart.cc/HariSekhon/DevOps-Python-tools)
+
diff --git a/ambari_ams_metrics.sh b/ambari_ams_metrics.sh
index 85f24ee53..64f20f0a2 100755
--- a/ambari_ams_metrics.sh
+++ b/ambari_ams_metrics.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-16 22:14:34 +0100 (Mon, 16 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# There were 2346 metrics last I checked but this probably varies a lot based on what services are deployed
diff --git a/ambari_blueprints.py b/ambari_blueprints.py
index 643bc3374..e5dd52217 100755
--- a/ambari_blueprints.py
+++ b/ambari_blueprints.py
@@ -1,18 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-08 14:09:50 +0000 (Sun, 08 Nov 2015)
# (re-instantiated from a Perl version in 2014)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -90,7 +90,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.10.2'
+__version__ = '0.10.3'
class AmbariBlueprintTool(CLI):
@@ -116,7 +116,7 @@ def connection(self, host, port, user, password, ssl=False, **kwargs):
#log.info("contacting Ambari as '%s'" % self.user)
if not isHost(host) or not isPort(port) or not isUser(user) or not password:
raise InvalidOptionException('invalid options passed to AmbariBlueprint()')
- proto = 'http' # pylint: disable=unused-variable
+ proto = 'http' # pylint: disable=unused-variable,possibly-unused-variable
if ssl:
proto = 'https'
self.host = host
@@ -140,8 +140,8 @@ def connection(self, host, port, user, password, ssl=False, **kwargs):
except IOError as _:
die("'failed to create dir '%s': %s" % (self.blueprint_dir, _))
- # TODO: change to @staticmethod
- def parse_cluster_name(self, item): # pylint: disable=no-self-use
+ @staticmethod
+ def parse_cluster_name(item):
if isStr(item):
item = json.loads(item)
try:
@@ -154,8 +154,8 @@ def get_clusters(self):
json_data = self.list('clusters')
return [self.parse_cluster_name(item) for item in json_data['items']]
- # TODO: change to @staticmethod
- def parse_blueprint_name(self, item): # pylint: disable=no-self-use
+ @staticmethod
+ def parse_blueprint_name(item):
if isStr(item):
item = json.loads(item)
try:
@@ -168,8 +168,8 @@ def get_blueprints(self):
json_data = self.list('blueprints')
return [self.parse_blueprint_name(item) for item in json_data['items']]
- # TODO: change to @staticmethod
- def parse_host_name(self, item): # pylint: disable=no-self-use
+ @staticmethod
+ def parse_host_name(item):
if isStr(item):
item = json.loads(item)
try:
@@ -320,7 +320,7 @@ def send_blueprint_file(self, filename, name=''):
except KeyError as _:
pass
if not name:
- name = os.path.splitext(os.path.basename(file))[0]
+ name = os.path.splitext(os.path.basename(filename))[0]
log.info("name not specified and couldn't determine blueprint name from blueprint data, reverting to using filename without extension '%s'" % name) # pylint: disable=line-too-long
# this solves the issue of having duplicate Blueprint.blueprint_name keys
try:
@@ -329,7 +329,7 @@ def send_blueprint_file(self, filename, name=''):
data = json.dumps(json_data)
log.info("reset blueprint field name to '%s'" % name)
except ValueError as _:
- qquit('CRITICAL', "invalid json found in file '%s': %s" % (file, name))
+ qquit('CRITICAL', "invalid json found in file '%s': %s" % (filename, name))
except KeyError as _:
log.warn('failed to reset the Blueprint name: %s' % _)
return self.send_blueprint(name, data)
@@ -399,8 +399,8 @@ def save_cluster(self, cluster, path=''):
log.debug("cluster '%s' blueprint content = '%s'" % (cluster, data))
self.save(cluster, path, data)
- # TODO: change to @staticmethod
- def save(self, name, path, data): # pylint: disable=no-self-use
+ @staticmethod
+ def save(name, path, data):
# log.debug('save(%s, %s)' % (name, data))
if data is None:
err = "blueprint '%s' returned None" % name
diff --git a/ambari_cancel_all_requests.sh b/ambari_cancel_all_requests.sh
index 15e079525..7d2df4425 100755
--- a/ambari_cancel_all_requests.sh
+++ b/ambari_cancel_all_requests.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-09-27 17:25:36 +0100 (Tue, 27 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/ambari_trigger_service_checks.py b/ambari_trigger_service_checks.py
index edf98b2bf..2afe64207 100755
--- a/ambari_trigger_service_checks.py
+++ b/ambari_trigger_service_checks.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-09-23 15:45:28 +0200 (Fri, 23 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/anonymize.py b/anonymize.py
index 74cb19226..ebcb20ab0 100755
--- a/anonymize.py
+++ b/anonymize.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# coding=utf-8
# vim:ts=4:sts=4:sw=4:et
#
@@ -6,16 +6,16 @@
# Date: 2018-08-08 19:02:02 +0100 (Wed, 08 Aug 2018)
# Original Date: 2013-07-18 21:17:41 +0100 (Thu, 18 Jul 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
-# ported from Perl version from DevOps Perl Tools repo (https://github.com/harisekhon/devops-perl-tools)
+# ported from Perl version from DevOps Perl Tools repo (https://github.com/HariSekhon/DevOps-Perl-tools)
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
@@ -36,7 +36,7 @@
Ignore phrases are in a similar file anonymize_ignore.conf, also adjacent to this program.
-Based on Perl Anonymize.pl from https://github.com/harisekhon/devops-perl-tools
+Based on Perl Anonymize.pl from https://github.com/HariSekhon/DevOps-Perl-tools
The Perl version is incredibly faster than Python due to the better regex engine
@@ -72,25 +72,30 @@
# pylint: disable=unused-import
from harisekhon.utils import \
aws_host_ip_regex, \
- domain_regex, \
domain_regex_strict, \
- email_regex, \
filename_regex, \
fqdn_regex, \
host_regex, \
hostname_regex, \
ip_prefix_regex, \
ip_regex, \
- mac_regex, \
subnet_mask_regex, \
user_regex
+ # used dynamically
+ # pylint: disable=unused-import
+ # lgtm [py/unused-import] - used by dynamic code so code analyzer cannot comprehend
+ from harisekhon.utils import \
+ domain_regex, \
+ email_regex, \
+ mac_regex \
+ # lgtm [py/unused-import] - used by dynamic code so code analyzer cannot comprehend
from harisekhon import CLI
except ImportError as _:
print(traceback.format_exc(), end='')
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.9.5'
+__version__ = '0.11.0'
ip_regex = r'(?!127\.0\.0\.)' + ip_regex
subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex
@@ -118,6 +123,10 @@ def __init__(self):
('ip', False),
('subnet_mask', False),
('mac', False),
+ ('db', False),
+ ('generic', False),
+ # kerberos must be applied before email
+ # - if email is applied first, 'user/host@realm' becomes 'user/', exposing user
('kerberos', False),
('email', False),
('password', False),
@@ -129,11 +138,12 @@ def __init__(self):
('screenos', False),
('junos', False),
('network', False),
+ ('windows', False),
+ ('aws', False), # access key, secret key, sts tokens etc are very generic so do them later
('fqdn', False),
('domain', False),
('hostname', False),
#('proxy', False),
- ('windows', False),
('custom', False),
])
self.exceptions = {
@@ -214,7 +224,7 @@ def __init__(self):
#'countryCode',
'displayName',
'displayNamePrintable',
- 'division'
+ 'division',
'employeeID',
'groupMembershipSAM',
'info',
@@ -281,7 +291,58 @@ def __init__(self):
arg_sep = r'[=\s:]+'
# openssl uses -passin switch
pass_word_phrase = r'(?:pass(?:word|phrase|in)?|userPassword)'
+ # allowing --blah- prefix variants
+ switch_prefix = r'(?. anyway by later fqdn anonymization
+ #'aws10': r'ec2-\d+-\d+-\d+-\d+\.{region}(\.compute\.amazonaws\.com)'.format(region='[A-Za-z0-9-]+'),
+ 'db': r'({switch_prefix}(?:db|database)-?name{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ switch_prefix=switch_prefix),
+ 'db2': r'({switch_prefix}(?:db|database)-?instance{id_or_name}?{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ id_or_name=id_or_name,
+ switch_prefix=switch_prefix),
+ 'db3': r'({switch_prefix}schema{id_or_name}?{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ id_or_name=id_or_name,
+ switch_prefix=switch_prefix),
+ 'db4': r'(\s(?:in|of|for)\s+(column|table|database|schema)[\s:]+[\'"])[^\'"]+',
+ 'db5': r'/+user/+hive/+warehouse/+([A-Za-z0-9_-]+/+)*[A-Za-z0-9_-]+.db/+[A-Za-z0-9_-]+',
+ 'generic': r'(\bfileb?)://{filename_regex}'.format(filename_regex=filename_regex),
+ 'generic2': r'({switch_prefix}key{id_or_name}?{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ id_or_name=id_or_name,
+ switch_prefix=switch_prefix),
+ 'generic3': r'({switch_prefix}cluster{id_or_name}?{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ id_or_name=id_or_name,
+ switch_prefix=switch_prefix),
+ 'generic4': r'({switch_prefix}function{id_or_name}?{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ id_or_name=id_or_name,
+ switch_prefix=switch_prefix),
+ 'generic5': r'({switch_prefix}load-?balancer{id_or_name}?{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ id_or_name=id_or_name,
+ switch_prefix=switch_prefix),
# don't change hostname or fqdn regex without updating hash_hostnames() option parse
# since that replaces these replacements and needs to match the grouping captures and surrounding format
'hostname2': r'({aws_host_ip})(?!-\d)'.format(aws_host_ip=aws_host_ip_regex),
@@ -297,17 +358,22 @@ def __init__(self):
'group2': r'({group_name}{sep}){user}'.format(group_name=group_name, sep=arg_sep, user=user_regex),
'group3': r'for\s+group\s+{group}'.format(group=user_regex),
'group4': r'(["\']{group_name}["\']\s*:\s*["\']?){group}'.format(group_name=group_name, group=user_regex),
+ 'group5': r'(arn:aws:iam:[^:]*:)\d+(:group/){group}'.format(
+ group='({user_regex}/)*{user_regex}'.format(user_regex=user_regex)),
'user': r'([-\.]{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep),
'user2': r'/(home|user)/{user}'.format(user=user_regex),
'user3': r'({user_name}{sep}){user}'.format(user_name=user_name, sep=arg_sep, user=user_regex),
- 'user4': r'(?/) exclude patterns '>/' where we have already matched and token replaced
'user6': r'(?/){user}@'.format(user=user_regex),
'user7': r'(["\'](?:{user_name}|owner)["\']\s*:\s*["\']?){user}'\
.format(user_name=user_name, user=user_regex),
- 'password': r'([-\.]?{pass_word_phrase}{sep}){pw}'\
+ #'user8': r'arn:aws:iam::\d{12}:user/{user}'.format(user=user_regex),
+ 'user8': r'(arn:aws:iam:[^:]*:)\d+(:user/){user}'.format(
+ user='({user_regex}/)*{user_regex}'.format(user_regex=user_regex)),
+ 'password': r'([\.-]?{pass_word_phrase}{sep}){pw}'\
.format(pass_word_phrase=pass_word_phrase,
sep=arg_sep,
pw=password_quoted),
@@ -317,6 +383,9 @@ def __init__(self):
sep=arg_sep,
pass_word_phrase=pass_word_phrase,
pw=password_quoted),
+ 'password4': r'([\.-]?(?:api-?)?token{sep}){pw}'\
+ .format(sep=arg_sep,
+ pw=password_quoted),
'ip': r'(?).*?$',
'cisco3': r'\ssecret\s.*?$',
'cisco4': r'\smd5\s+.*?$',
'cisco5': r'\scommunity\s+.*$',
@@ -380,6 +449,29 @@ def __init__(self):
ldap_lambda_lowercase = lambda m: r'{}<{}>'.format(m.group(1), m.group(2).lower())
# will auto-infer replacements to not have to be explicit, use this only for override mappings
self.replacements = {
+ # arn:partition:service:region:account-id:resource-id
+ # arn:partition:service:region:account-id:resource-type/resource-id
+ # arn:partition:service:region:account-id:resource-type:resource-id
+ 'aws': r'\1\2<\3>',
+ 'aws2': r'\1:',
+ 'aws3': r'',
+ 'aws4': r'',
+ 'aws5': r'',
+ 'aws6': r'',
+ 'aws7': r'',
+ 'aws8': r'\1:///',
+ 'aws9': r'',
+ #'aws10': r'ec2-x-x-x-x.\1',
+ 'db': r'\1',
+ 'db2': r'\1',
+ 'db3': r'\1',
+ 'db4': r'\1<\2>',
+ 'db5': r'/user/hive/warehouse/.db/',
+ 'generic': r'\1://',
+ 'generic2': r'\1',
+ 'generic3': r'\1',
+ 'generic4': r'\1',
+ 'generic5': r'\1',
'hostname': r':\2',
#'hostname2': '',
'hostname2': r'',
@@ -396,13 +488,16 @@ def __init__(self):
'user5': 'for user ',
'user6': '@',
'user7': r'\1',
+ 'user8': r'\1\2',
'group': r'\1',
'group2': r'\1',
'group3': r'for group ',
'group4': r'\1',
+ 'group5': r'\1\2',
'password': r'\1',
'password2': r'\1:',
'password3': r'\1\2',
+ 'password4': r'\1',
'ip': r'/',
'ip2': r'',
'ip3': r'',
@@ -456,6 +551,12 @@ def add_options(self):
self.add_opt('-a', '--all', action='store_true',
help='Apply all anonymizations (careful this includes --host which can be overzealous and ' + \
'match too many things, in which case try more targeted anonymizations below)')
+ self.add_opt('-w', '--aws', action='store_true',
+ help='Apply AWS anonymizations (access/secret keys, STS tokens, ARNs, buckets, security groups)')
+ self.add_opt('-b', '--db', '--database', action='store_true',
+ help='Apply database anonymizations (db name, instance name)')
+ self.add_opt('-g', '--generic', action='store_true',
+ help='Apply generic anonymizations (file://, key, cluster name / id etc)')
self.add_opt('-C', '--custom', action='store_true',
help='Apply custom phrase anonymization (add your Name, Company Name etc to the list of ' + \
'blacklisted words/phrases one per line in anonymize_custom.conf). Matching is case ' + \
@@ -484,7 +585,7 @@ def add_options(self):
'debugging, these are salted and truncated to be indistinguishable from temporal docker ' + \
'container IDs, but someone with enough computing power and time could theoretically ' + \
'calculate the source hostnames so don\'t put these on the public internet, it is more ' + \
- 'for private vendor tickets)')
+ 'for private vendor tickets')
self.add_opt('-d', '--domain', action='store_true',
help='Apply domain format anonymization')
self.add_opt('-F', '--fqdn', action='store_true',
@@ -503,16 +604,11 @@ def add_options(self):
r'http://username:password\@ => ' + \
r'http://:\@. Also works with https://')
self.add_opt('-K', '--kerberos', action='store_true',
- help=r'Kerberos 5 principals in the form @ or /@ ' + \
+ help=r'Apply Kerberos anonymizations eg. @, /@ ' + \
'(where must match a valid domain name - otherwise use --custom and populate ' + \
- r'anonymize_custom.conf). These kerberos principals are anonymizebed to ' + \
- '. There is a special exemption for Hadoop Kerberos principals such ' + \
- 'as NN/_HOST@ which preserves the literal \'_HOST\' instance since that\'s ' + \
- 'useful to know for debugging, the principal and realm will still be anonymizebed in ' + \
- 'those cases (if wanting to retain NN/_HOST then use --domain instead of --kerberos). ' + \
- 'This is applied before --email in order to not prevent the email replacement leaving ' + \
- r'this as user/host\@realm to user/, which would have exposed \'user\'' + \
- '. Auto enables --email, --domain and --fqdn')
+ r'anonymize_custom.conf). Hadoop principals preserve the generic _HOST placeholder eg. ' + \
+ '/_HOST@ (if wanting to retain full prefix eg. NN/_HOST then use ' + \
+ '--domain instead of --kerberos). --kerberos auto-enables --email, --domain and --fqdn')
self.add_opt('-L', '--ldap', action='store_true',
help='Apply LDAP anonymization ' + \
'(~100 attribs eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...)')
@@ -523,7 +619,7 @@ def add_options(self):
# 'should probably also apply --ip and --host if using this. Auto enables --http-auth')
self.add_opt('-N', '--network', action='store_true',
help='Apply all network anonymization, whether Cisco, ScreenOS, JunOS for secrets, auth, ' + \
- 'usernames, passwords, md5s, PSKs, AS, SNMP etc.')
+ 'usernames, passwords, md5s, PSKs, AS, SNMP community strings etc.')
self.add_opt('-c', '--cisco', action='store_true',
help='Apply Cisco IOS/IOS-XR/NX-OS configuration format anonymization')
self.add_opt('-s', '--screenos', action='store_true',
@@ -564,9 +660,14 @@ def process_options(self):
self.anonymizations['ip'] = False
else:
for _ in self.anonymizations:
+ # pylint: disable=no-else-continue
if _ in ('subnet_mask', 'mac', 'group'):
continue
- self.anonymizations[_] = self.get_opt(_)
+ elif _ == 'database':
+ self.anonymizations['db'] = True
+ else:
+ self.anonymizations[_] = self.get_opt(_)
+ log.debug('anonymization enabled %s = %s', _, bool(self.anonymizations[_]))
self._process_options_host()
self._process_options_network()
self._process_options_exceptions()
@@ -691,46 +792,47 @@ def run(self):
# allow to easily switch pre-compilation on/off for testing
# testing shows on a moderate sized file that it is a couple secs quicker to use pre-compiled regex
def compile(self, name, regex):
+ log.debug(f"compiling regex '{name}' = '{regex}'")
self.regex[name] = re.compile(regex, re.I)
#self.regex[name] = regex
def prepare_regex(self):
self.compile('hostname',
- r'(? 2018-01-:00:00
- r'(?!\d+T\d+:\d+)' + \
- r'(?!\d+[^A-Za-z0-9]|' + \
- self.custom_ignores_raw + ')' + \
- '(' + hostname_regex + ')' + \
+ r'(?!\d+T\d+:\d+)' +
+ r'(?!\d+[^A-Za-z0-9]|' +
+ self.custom_ignores_raw + ')' +
+ '(' + hostname_regex + ')' +
self.negative_host_lookbehind + r':(\d{1,5}(?!\.?\w))',
)
self.compile('domain',
# don't match java -some.net.property
- #r'(?/dev/null || :
echo
- "$srcdir/bash-tools/split.sh" --parts "$parallelism" "$filename"
+ "$srcdir/bash-tools/bin/split.sh" --parts "$parallelism" "$filename"
echo "Anonymizing parts"
for file_part in "$filename".*; do
cmd="$srcdir/anonymize.py -a $file_part > $file_part.anonymized"
diff --git a/aws_s3_presign.py b/aws_s3_presign.py
new file mode 100755
index 000000000..e1addfbbc
--- /dev/null
+++ b/aws_s3_presign.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-01-14 17:45:38 +0000 (Tue, 14 Jan 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Generate and print an S3 pre-signed URL
+
+Can do the same with the following AWS CLI command:
+
+aws s3 presign s3:/// [--expires-in ]
+
+Will generate a pre-signed URL even when the bucket and object key don't exist!
+
+(you will get a runtime error when requesting the link that the bucket or object doesn't exist)
+
+
+Uses the Boto library, read here for the list of ways to configure your AWS credentials:
+
+ https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
+import sys
+import boto3
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.1'
+
+def main():
+ parser = argparse.ArgumentParser(
+ description='Generate an AWS S3 pre-signed URL to access an S3 object without logging in')
+ parser.add_argument('bucket', help='Bucket Name')
+ parser.add_argument('key', help='Key')
+ parser.add_argument('expiration', nargs='?', default=3600, help='Expiration of URL in seconds')
+ args = parser.parse_args()
+
+ # more useful if doing this programmatically as we can do this on the command line via AWS CLI
+ conn = boto3.client('s3')
+ url = conn.generate_presigned_url(
+ 'get_object',
+ Params={
+ 'Bucket': args.bucket,
+ 'Key': args.key
+ },
+ ExpiresIn=args.expiration
+ )
+ print(url)
+
+
+if __name__ == '__main__':
+ try:
+ main()
+ except KeyboardInterrupt:
+ print('Control-C...', file=sys.stderr)
diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py
new file mode 100755
index 000000000..be218c2db
--- /dev/null
+++ b/aws_users_access_key_age.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-12-13 17:24:40 +0000 (Fri, 13 Dec 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Lists all AWS IAM users keys along with their ages, optionally filtering any older than a given number of days
+
+Output format is:
+
+
+
+Status is Active or Inactive
+
+Validated compared to xls report download from Trusted Advisor -> Security -> IAM Access Key Rotation
+
+Uses the Boto library, read here for the list of ways to configure your AWS credentials:
+
+ https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
+
+This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies
+
+See also:
+
+ aws_users_access_key_age.sh - simpler version in the adjacent DevOps Bash Tools repo without age filtering
+ - https://github.com/HariSekhon/DevOps-Bash-tools
+
+Advanced Nagios Plugins (https://github.com/HariSekhon/Nagios-Plugins)
+
+ check_aws_access_keys_age.py
+ check_aws_access_keys_disabled.py
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import datetime
+import os
+import sys
+from math import ceil
+import boto3
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_float
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.3.0'
+
+class AWSUsersAccessKeysAge(CLI):
+
+ def __init__(self):
+ super(AWSUsersAccessKeysAge, self).__init__()
+ self.age = None
+ self.now = None
+ self.only_active_keys = False
+ self.timeout_default = 300
+
+ def add_options(self):
+ self.add_opt('-a', '--age', help='Return keys older than N days')
+ self.add_opt('-o', '--only-active', action='store_true', help='Return only keys with Active status')
+
+ def process_args(self):
+ self.only_active_keys = self.get_opt('only_active')
+ self.age = self.get_opt('age')
+ if self.age:
+ validate_float(self.age, 'age')
+ self.age = float(self.age)
+
+ def run(self):
+ iam = boto3.client('iam')
+ user_paginator = iam.get_paginator('list_users')
+ self.now = datetime.datetime.utcnow()
+ for users_response in user_paginator.paginate():
+ for user_item in users_response['Users']:
+ username = user_item['UserName']
+ key_paginator = iam.get_paginator('list_access_keys')
+ for keys_response in key_paginator.paginate(UserName=username):
+ self.process_key(keys_response, username)
+ log.info('Completed')
+
+ def process_key(self, keys_response, username):
+ #assert not keys_response['IsTruncated']
+ for access_key_item in keys_response['AccessKeyMetadata']:
+ assert username == access_key_item['UserName']
+ status = access_key_item['Status']
+ if self.only_active_keys and status != 'Active':
+ continue
+ create_date = access_key_item['CreateDate']
+ # already cast to datetime.datetime with tzinfo
+ #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%dT%H:%M:%S%z')
+ # removing tzinfo for comparison to avoid below error
+ # - both areOA UTC and this doesn't make much difference anyway
+ # TypeError: can't subtract offset-naive and offset-aware datetimes
+ age_timedelta = self.now - create_date.replace(tzinfo=None)
+ age_days = int(ceil(age_timedelta.total_seconds() / 86400.0))
+ if self.age:
+ if age_days < self.age:
+ continue
+ print('{user:20}\t{status:8}\t{date}\t({days:>3} days)'.format(
+ user=username,
+ status=status,
+ date=create_date,
+ days=age_days))
+
+
+if __name__ == '__main__':
+ AWSUsersAccessKeysAge().main()
diff --git a/aws_users_last_used.py b/aws_users_last_used.py
new file mode 100755
index 000000000..e06a7bcdb
--- /dev/null
+++ b/aws_users_last_used.py
@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-12-16 11:37:15 +0000 (Mon, 16 Dec 2019)
+#
+# https://github.com/HariSekhon/Nagios-Plugins
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Find AWS IAM user accounts last used age in days using the most recently used among their timestamp and access keys
+
+Optionally filters to only users > N days old to find old user accounts
+
+Generates an IAM credential report, then parses it to determine the time since each user's password
+and access keys were last used
+
+Requires iam:GenerateCredentialReport on resource: *
+
+Output:
+
+
+
+Uses the Boto python library, read here for the list of ways to configure your AWS credentials:
+
+ https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
+
+See also the DevOps Bash Tools and Advanced Nagios Plugins Collection repos which have more similar AWS tools
+
+- https://github.com/HariSekhon/DevOps-Bash-tools
+- https://github.com/HariSekhon/Nagios-Plugins
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import csv
+import os
+import sys
+import time
+import traceback
+from datetime import datetime
+from io import StringIO
+from math import floor
+import boto3
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_int
+ from harisekhon import CLI
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.1'
+
+
+class AWSUsersLastUsed(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(AWSUsersLastUsed, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.age = None
+ self.now = None
+ self.timeout_default = 300
+ self.msg = 'AWSUsersLastUsed msg not defined'
+
+ def add_options(self):
+ self.add_opt('-a', '--age', type=float,
+ help='Filters to show only accounts last used more than N days ago')
+
+ def process_args(self):
+ self.no_args()
+ self.age = self.get_opt('age')
+ if self.age is not None:
+ validate_int(self.age, 'age')
+
+ def run(self):
+ iam = boto3.client('iam')
+ log.info('generating credentials report')
+ while True:
+ result = iam.generate_credential_report()
+ log.debug('%s', result)
+ if result['State'] == 'COMPLETE':
+ log.info('credentials report generated')
+ break
+ log.info('waiting for credentials report')
+ time.sleep(1)
+ #try:
+ result = iam.get_credential_report()
+ #except ClientError as _:
+ # raise
+ csv_content = result['Content']
+ log.debug('%s', csv_content)
+ filehandle = StringIO(unicode(csv_content))
+ filehandle.seek(0)
+ csvreader = csv.reader(filehandle)
+ headers = next(csvreader)
+ assert headers[0] == 'user'
+ assert headers[4] == 'password_last_used'
+ assert headers[10] == 'access_key_1_last_used_date'
+ assert headers[15] == 'access_key_2_last_used_date'
+ self.now = datetime.utcnow()
+ for row in csvreader:
+ self.process_user(row)
+
+ def process_user(self, row):
+ log.debug('processing user: %s', row)
+ user = row[0]
+ password_last_used = row[4]
+ access_key_1_last_used_date = row[10]
+ access_key_2_last_used_date = row[15]
+ log.debug('user: %s, password_last_used: %s, access_key_1_last_used_date: %s, access_key_2_last_used_date: %s',
+ user, password_last_used, access_key_1_last_used_date, access_key_2_last_used_date)
+ min_age = None
+ for _ in [password_last_used, access_key_1_last_used_date, access_key_2_last_used_date]:
+ if _ == 'N/A':
+ continue
+ # %z not working in Python 2.7 but we already know it's +00:00
+ _datetime = datetime.strptime(_.split('+')[0], '%Y-%m-%dT%H:%M:%S')
+ age_timedelta = self.now - _datetime.replace(tzinfo=None)
+ age_days = int(floor(age_timedelta.total_seconds() / 86400.0))
+ if min_age is None or age_days < min_age:
+ min_age = age_days
+ if self.age and min_age <= self.age:
+ return
+ print('{user:20}\t{days:>3}\t{password_last_used:25}\t'\
+ .format(user=user,
+ days=min_age,
+ password_last_used=password_last_used) +
+ '{access_key_1_last_used_date:25}\t{access_key_2_last_used_date:25}'\
+ .format(access_key_1_last_used_date=access_key_1_last_used_date,
+ access_key_2_last_used_date=access_key_2_last_used_date))
+
+
+if __name__ == '__main__':
+ AWSUsersLastUsed().main()
diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py
new file mode 100755
index 000000000..6810abd17
--- /dev/null
+++ b/aws_users_pw_last_used.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-12-19 11:43:25 +0000 (Thu, 19 Dec 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Lists all AWS IAM users dates since their passwords were last used, optionally filtering
+for users whose passwords haven't been used in > N days
+
+Output format is:
+
+
+
+Uses Boto, read here for the list of ways to configure your AWS credentials:
+
+ https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
+
+This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies
+
+See also the DevOps Bash Tools repo and The Advanced Nagios Plugins Collection for similar tools
+
+https://github.com/HariSekhon/DevOps-Bash-tools
+
+https://github.com/HariSekhon/Nagios-Plugins
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import datetime
+import os
+import sys
+from math import floor
+import boto3
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_float
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+class AWSUsersPasswordLastUsed(CLI):
+
+ def __init__(self):
+ super(AWSUsersPasswordLastUsed, self).__init__()
+ self.age = None
+ self.now = None
+ self.timeout_default = 300
+
+ def add_options(self):
+ self.add_opt('-a', '--age', help='Return users with passwords last used more than N days ago')
+
+ def process_args(self):
+ self.age = self.get_opt('age')
+ if self.age:
+ validate_float(self.age, 'age')
+ self.age = float(self.age)
+
+ def run(self):
+ iam = boto3.client('iam')
+ user_paginator = iam.get_paginator('list_users')
+ self.now = datetime.datetime.utcnow()
+ for users_response in user_paginator.paginate():
+ for user_item in users_response['Users']:
+ log.debug(user_item)
+ self.process_password_last_used(user_item)
+ log.info('Completed')
+
+ def process_password_last_used(self, user_item):
+ # already cast to datetime.datetime with tzinfo
+ user = user_item['UserName']
+ if 'PasswordLastUsed' in user_item:
+ password_last_used = user_item['PasswordLastUsed']
+ # removing tzinfo for comparison to avoid below error
+ # - both are UTC and this doesn't make much difference anyway
+ # TypeError: can't subtract offset-naive and offset-aware datetimes
+ datetime_delta = self.now - password_last_used.replace(tzinfo=None)
+ days = int(floor(datetime_delta.total_seconds() / 86400))
+ if self.age and days <= self.age:
+ return
+ else:
+ password_last_used = 'N/A'
+ days = 'N/A'
+ print('{user:20s}\t{password_last_used:25s}\t({days:>3} days)'.format(
+ user=user,
+ password_last_used=str(password_last_used), # without str() format string breaks with :25
+ days=days))
+
+
+if __name__ == '__main__':
+ AWSUsersPasswordLastUsed().main()
diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py
new file mode 100755
index 000000000..befb152ee
--- /dev/null
+++ b/aws_users_unused_access_keys.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-12-16 11:37:15 +0000 (Mon, 16 Dec 2019)
+#
+# https://github.com/HariSekhon/Nagios-Plugins
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Find AWS IAM user access keys that haven't been used in N days and keys that have never been used
+
+Generates an IAM credential report, then parses it to determine the time since each user's password
+and access keys were last used
+
+Requires iam:GenerateCredentialReport on resource: *
+
+Output:
+
+
+
+Uses the Boto python library, read here for the list of ways to configure your AWS credentials:
+
+ https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
+
+See also the DevOps Bash Tools and Advanced Nagios Plugins Collection repos which have more similar AWS tools
+
+- https://github.com/HariSekhon/DevOps-Bash-tools
+- https://github.com/HariSekhon/Nagios-Plugins
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import csv
+import os
+import sys
+import time
+import traceback
+from datetime import datetime
+from io import StringIO
+from math import floor
+import boto3
+from botocore.exceptions import ClientError
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_int
+ from harisekhon import CLI
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.2.1'
+
+
+class AWSUnusedAccessKeys(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(AWSUnusedAccessKeys, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.age = None
+ self.now = None
+ self.only_active = False
+ self.timeout_default = 300
+ self.msg = 'AWSUnusedAccessKeys msg not defined'
+
+ def add_options(self):
+ self.add_opt('-a', '--age', type=float, default=30,
+ help='Show only access keys not used in the last N days (default: 30)')
+ self.add_opt('-o', '--only-active', action='store_true',
+ help='Only show access keys that are active')
+
+ def process_args(self):
+ self.no_args()
+ self.age = self.get_opt('age')
+ if self.age is not None:
+ validate_int(self.age, 'age')
+
+ def run(self):
+ iam = boto3.client('iam')
+ log.info('generating credentials report')
+ while True:
+ result = iam.generate_credential_report()
+ log.debug('%s', result)
+ if result['State'] == 'COMPLETE':
+ log.info('credentials report generated')
+ break
+ log.info('waiting for credentials report')
+ time.sleep(1)
+ try:
+ result = iam.get_credential_report()
+ except ClientError as _:
+ raise
+ csv_content = result['Content']
+ log.debug('%s', csv_content)
+ filehandle = StringIO(unicode(csv_content))
+ filehandle.seek(0)
+ csvreader = csv.reader(filehandle)
+ headers = next(csvreader)
+ log.debug('headers: %s', headers)
+ assert headers[0] == 'user'
+ assert headers[8] == 'access_key_1_active'
+ assert headers[9] == 'access_key_1_last_rotated'
+ assert headers[10] == 'access_key_1_last_used_date'
+ assert headers[13] == 'access_key_2_active'
+ assert headers[14] == 'access_key_2_last_rotated'
+ assert headers[15] == 'access_key_2_last_used_date'
+ self.now = datetime.utcnow()
+ for row in csvreader:
+ self.process_user(row)
+
+ def process_user(self, row):
+ log.debug('processing user: %s', row)
+ user = row[0]
+ access_keys = {1:{}, 2:{}}
+ access_keys[1]['active'] = row[8]
+ access_keys[1]['last_used_date'] = row[10]
+ access_keys[1]['last_rotated'] = row[9]
+ access_keys[2]['active'] = row[13]
+ access_keys[2]['last_rotated'] = row[14]
+ access_keys[2]['last_used_date'] = row[15]
+ for key in [1, 2]:
+ active = access_keys[key]['active']
+ if not isinstance(active, bool):
+ assert active in ('true', 'false')
+ active = active.lower() == 'true'
+ created = access_keys[key]['last_rotated']
+ last_used = access_keys[key]['last_used_date']
+ log.debug('user: %s, key: %s, active: %s, created: %s, last_used_date: %s, ',
+ user,
+ key,
+ active,
+ created,
+ last_used
+ )
+ if not active and self.only_active:
+ continue
+ if last_used == 'N/A':
+ if created == 'N/A':
+ continue
+ self.print_key(user, key, active, 'N/A', last_used, created)
+ continue
+ # %z not working in Python 2.7 but we already know it's +00:00
+ _datetime = datetime.strptime(last_used.split('+')[0], '%Y-%m-%dT%H:%M:%S')
+ age_timedelta = self.now - _datetime.replace(tzinfo=None)
+ age_days = int(floor(age_timedelta.total_seconds() / 86400.0))
+ if age_days > self.age:
+ self.print_key(user, key, active, age_days, last_used, created)
+
+ # pylint: disable=too-many-arguments
+ def print_key(self, user, key, active, age_days, last_used, created):
+ log.debug('access key %s, active: %s, last used %s days ago > %s', key, active, age_days, self.age)
+ print('{user:20}\t{key}\t{active}\t{days:>3}\t{last_used:25}\t{created}'\
+ .format(user=user,
+ key=key,
+ active='Active' if active else 'Inactive',
+ days=age_days,
+ last_used=last_used,
+ created=created
+ )
+ )
+
+
+if __name__ == '__main__':
+ AWSUnusedAccessKeys().main()
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
new file mode 100644
index 000000000..b430c58dd
--- /dev/null
+++ b/azure-pipelines.yml
@@ -0,0 +1,70 @@
+# vim:ts=2:sts=2:sw=2:et
+#
+# Author: Hari Sekhon
+# Date: Sun Feb 23 19:02:10 2020 +0000
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# A z u r e D e v O p s P i p e l i n e
+# ============================================================================ #
+
+# https://aka.ms/yaml
+
+trigger:
+ - master
+
+variables:
+ # ubuntu version
+ os_version: '22.04'
+
+pool:
+ # there is no /dev/stderr on this azure build!
+ #vmImage: 'ubuntu-latest'
+ #vmImage: 'ubuntu-22.04'
+ vmImage: 'ubuntu-$(os_version)'
+
+# unprivileged container without sudo, cannot install dependencies
+#container: ubuntu:22.04
+
+steps:
+ - script: cat /etc/*-release
+ displayName: OS Release
+
+ # requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName'
+ - script: env | sort
+ displayName: Environment
+
+ # doesn't work in container due to unprivileged execution and lack of sudo
+ #- script: sudo apt-get update && sudo apt-get install -y git make
+ # displayName: install git & make
+
+ #- script: make
+ # displayName: build
+
+ # doesn't work in vmImage build due to lack of access to normal /dev/stderr device
+ # tee: /dev/stderr: No such device or address
+ #- script: make test
+ # displayName: test
+
+ # hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-(
+ - script: |
+ sudo docker run -v "$PWD":/code "ubuntu:$(os_version)" /bin/bash -c '
+ set -ex
+ cd /code
+ setup/ci_bootstrap.sh
+ if [ -x setup/ci_git_set_dir_safe.sh ]; then
+ setup/ci_git_set_dir_safe.sh
+ fi
+ make init
+ make ci test
+ '
+ displayName: docker build
diff --git a/bash-tools b/bash-tools
index cea4c6fae..11dee29ce 160000
--- a/bash-tools
+++ b/bash-tools
@@ -1 +1 @@
-Subproject commit cea4c6fae729d43015eb861fbf52f03abe52f2c0
+Subproject commit 11dee29cea607445270d8bd675d1bcdecb069c14
diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml
new file mode 100644
index 000000000..cdf5243ee
--- /dev/null
+++ b/bitbucket-pipelines.yml
@@ -0,0 +1,38 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-24 17:08:57 +0000 (Mon, 24 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# B i t b u c k e t C I / C D P i p e l i n e
+# ============================================================================ #
+
+# Reference:
+#
+# https://support.atlassian.com/bitbucket-cloud/docs/configure-bitbucket-pipelinesyml/
+
+# Languages:
+#
+# https://confluence.atlassian.com/x/5Q4SMw
+
+# You can specify a custom docker image from Docker Hub as your build environment.
+image: atlassian/default-image:2
+
+pipelines:
+ default:
+ - step:
+ script:
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci
+ - make test
diff --git a/boot b/boot
new file mode 120000
index 000000000..4092e5539
--- /dev/null
+++ b/boot
@@ -0,0 +1 @@
+setup/bootstrap.sh
\ No newline at end of file
diff --git a/buddy.yml b/buddy.yml
new file mode 100644
index 000000000..e0b57451f
--- /dev/null
+++ b/buddy.yml
@@ -0,0 +1,48 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-03-16 14:02:53 +0000 (Mon, 16 Mar 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# B u d d y C I
+# ============================================================================ #
+
+# https://buddy.works/docs/yaml/yaml-schema
+
+---
+- pipeline: "Build"
+ trigger_mode: "ON_EVERY_PUSH"
+ ref_name: "master"
+ ref_type: "BRANCH"
+ target_site_url: "https://github.com/HariSekhon/DevOps-Python-tools"
+ trigger_condition: "ALWAYS"
+ actions:
+ - action: "Execute: make ci test"
+ type: "BUILD"
+ working_directory: "/buddy/DevOps-Python-tools"
+ docker_image_name: "library/ubuntu"
+ docker_image_tag: "18.04"
+ #setup_commands:
+ # this step gets cached, which results in
+ # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?
+ # - apt update
+ # - apt install -qy git make
+ execute_commands:
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci
+ - make test
+ volume_mappings:
+ - "/:/buddy/DevOps-Python-tools"
+ shell: "BASH"
+ trigger_condition: "ALWAYS"
diff --git a/center.py b/center.py
index 752db7f6a..379353e68 100755
--- a/center.py
+++ b/center.py
@@ -1,17 +1,19 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
+# args: -s <<< "Auth & Config"
+# args: -s <<< "GKE Clusters"
#
# Author: Hari Sekhon
# Date: 2016-01-29 21:05:38 +0000 (Fri, 29 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -42,7 +44,8 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.4.0'
+__version__ = '0.5.1'
+
class Center(CLI):
@@ -51,8 +54,13 @@ def __init__(self):
super(Center, self).__init__()
# Python 3.x
# super().__init__()
- self.re_bound = re.compile(r'(\b)')
+ # this doesn't put enough spaces around ampersands, eg. in "Auth & Config"
+ #self.re_bound = re.compile(r'(\b)')
+ self.re_spaces = re.compile(r'(\s)')
+ self.re_multiple_spaces = re.compile(r'(\s){2}')
self.re_chars = re.compile(r'([^\s])(?!\s)')
+ self.re_chars_spaced = re.compile(r'([^\s])\s')
+ self.timeout_default = None
def add_options(self):
self.add_opt('-w', '--width', default=80, type='int', metavar='',
@@ -61,11 +69,16 @@ def add_options(self):
help='No comment prefix handling')
self.add_opt('-s', '--space', action='store_true', default=False,
help='Space all chars out, makes bigger headings')
+ self.add_opt('-u', '--unspace', action='store_true', default=False,
+ help='Removes spaces betweeen chars out, the inverse of --space')
def run(self):
log_option('width', self.get_opt('width'))
log_option('no comment prefix', self.get_opt('no_comment'))
log_option('space chars', self.get_opt('space'))
+ log_option('unspace chars', self.get_opt('unspace'))
+ if self.get_opt('space') and self.get_opt('unspace'):
+ self.usage("--space and --unspace are mutually exclusive!")
if self.args:
self.process_line(' '.join(self.args))
else:
@@ -73,10 +86,16 @@ def run(self):
self.process_line(line)
def space(self, line):
- line = self.re_bound.sub(r' ', line)
+ #line = self.re_bound.sub(r' ', line)
+ line = self.re_spaces.sub(r'\1\1\1', line)
line = self.re_chars.sub(r'\1 ', line)
return line
+ def unspace(self, line):
+ line = self.re_chars_spaced.sub(r'\1', line)
+ line = self.re_multiple_spaces.sub(r'\1', line)
+ return line
+
def process_line(self, line):
char = ''
if not line:
@@ -85,7 +104,7 @@ def process_line(self, line):
char = ' '
# preliminary strip() to be able to pick up # if it isn't the first char and their are spaces before it
line = line.strip()
- if isChars(line[0], '#'):
+ if line and isChars(line[0], '#'):
char = line[0]
line = line.lstrip(char)
elif len(line) > 1 and isChars(line[0:1], '/'):
@@ -96,9 +115,12 @@ def process_line(self, line):
line = line.lstrip(char)
if self.get_opt('space'):
line = self.space(line)
+ if self.get_opt('unspace'):
+ line = self.unspace(line)
line = line.strip()
side = int(max((self.get_opt('width') - len(line)) / 2, 0))
print(char + ' ' * side + line)
+
if __name__ == '__main__':
Center().main()
diff --git a/cicd/.concourse.yml b/cicd/.concourse.yml
new file mode 100644
index 000000000..92847121e
--- /dev/null
+++ b/cicd/.concourse.yml
@@ -0,0 +1,65 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-03-21 11:06:48 +0000 (Sat, 21 Mar 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C o n c o u r s e C I
+# ============================================================================ #
+
+# https://concourse-ci.org/golang-library-example.html
+
+# https://resource-types.concourse-ci.org/
+# https://concourse-ci.org/resource-types.html
+resources:
+ - name: github
+ icon: github-circle
+ type: git
+ source:
+ uri: https://github.com/HariSekhon/DevOps-Python-tools
+ branch: master
+ #- name: daily
+ # type: time
+ # source:
+ # interval: 1d
+
+# https://concourse-ci.org/jobs.html
+jobs:
+ - name: build
+ public: false
+ plan:
+ - get: github # from resource above
+ trigger: true
+ #version: every # build every git commit, default: latest
+ - task: build
+ config:
+ platform: linux
+ image_resource:
+ type: docker-image
+ source:
+ repository: ubuntu
+ tag: latest
+ inputs:
+ - name: github
+ path: code
+ params:
+ CONCOURSE: 1
+ run:
+ path: /bin/bash
+ args:
+ - -c
+ - |
+ cd code &&
+ setup/ci_bootstrap.sh &&
+ make init &&
+ make ci test
diff --git a/cicd/.gocd.yml b/cicd/.gocd.yml
new file mode 100644
index 000000000..65583a818
--- /dev/null
+++ b/cicd/.gocd.yml
@@ -0,0 +1,94 @@
+# vim:ts=2:sts=2:sw=2:et
+#
+# Author: Hari Sekhon
+# Date: 2020-03-21 11:14:07 +0000 (Sat, 21 Mar 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# G o C D
+# ============================================================================ #
+
+# https://github.com/tomzo/gocd-yaml-config-plugin#setup
+
+# https://docs.gocd.org/current/configuration/configuration_reference.html
+
+---
+format_version: 3
+pipelines:
+ DevOps-Python-tools:
+ group: defaultGroup
+ label_template: ${COUNT}
+ lock_behavior: none
+ display_order: -1
+ materials:
+ git:
+ git: https://github.com/HariSekhon/DevOps-Python-tools
+ shallow_clone: false
+ auto_update: true
+ branch: master
+ stages:
+ - build-and-test:
+ fetch_materials: true
+ keep_artifacts: false
+ clean_workspace: false
+ approval:
+ type: success
+ allow_only_on_success: false
+ jobs:
+ #apt-update:
+ # timeout: 10
+ # tasks:
+ # - exec:
+ # command: apt
+ # arguments:
+ # - update
+ # run_if: passed
+ #install-make:
+ # timeout: 10
+ # tasks:
+ # - exec:
+ # command: apt
+ # arguments:
+ # - install
+ # - -qy
+ # - git
+ # - make
+ # run_if: passed
+ ci-bootstrap:
+ timeout: 10
+ tasks:
+ - exec:
+ command: setup/ci_bootstrap.sh
+ run_if: passed
+ init:
+ timeout: 10
+ tasks:
+ - exec:
+ command: make
+ arguments:
+ - init
+ run_if: passed
+ build:
+ timeout: 60
+ tasks:
+ - exec:
+ command: make
+ arguments:
+ - ci
+ run_if: passed
+ test:
+ timeout: 60
+ tasks:
+ - exec:
+ command: make
+ arguments:
+ - test
+ run_if: passed
diff --git a/cicd/buildspec.yml b/cicd/buildspec.yml
new file mode 100644
index 000000000..f4e887953
--- /dev/null
+++ b/cicd/buildspec.yml
@@ -0,0 +1,66 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-12-19 15:32:28 +0000 (Sat, 19 Dec 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# A W S C o d e B u i l d
+# ============================================================================ #
+
+# References:
+#
+# https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html
+
+# Operating System should be set to Ubuntu, not Amazon Linux 2
+# - this is both recommended since programming language runtimes are now included in standard image of ubuntu, but also to avoid this error:
+#
+# /usr/bin/amazon-linux-extras
+# /root/.pyenv/versions/3.8.3/bin/python: No module named amazon_linux_extras
+
+version: 0.2
+
+# only on Linux, the user to run as - global setting, alternatively set inside a phase section below for localized user
+#run-as: linux-username
+
+env:
+ shell: bash
+ # don't store sensitive stuff like AWS secret keys in variables, use parameter-store or secrets-manager
+ # any environment variables replace existing environment variables, ie. beware if setting PATH that it'll replace the existing PATH with a non-interpolated literal
+ # project env vars take precedence over these, with start build vars taking highest precedence
+ #variables:
+ # DEBUG: "1"
+ #exported-variables:
+ # - DEBUG
+
+phases:
+ # install prerequisites / languages / frameworks / packages to allow build to work
+ install:
+ #commands:
+ # - setup/ci_bootstrap.sh
+ # languages to install
+ runtime-versions:
+ #java: openjdk11
+ # AWS LTS release of OpenJDK 11
+ java: corretto11
+ golang: 1.14
+ python: 3.8
+ ruby: 2.7
+ # eg. sign in to Amazon ECR or install package dependencies
+ pre_build:
+ commands:
+ - setup/ci_bootstrap.sh
+ build:
+ commands:
+ - echo Build started on `date`
+ - make
+ - echo Build completed on `date`
+ - make test
+ - echo Tests completed on `date`
diff --git a/cicd/cloudbuild.yaml b/cicd/cloudbuild.yaml
new file mode 100644
index 000000000..61e439fa3
--- /dev/null
+++ b/cicd/cloudbuild.yaml
@@ -0,0 +1,41 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-12-19 16:27:26 +0000 (Sat, 19 Dec 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# G C P C l o u d B u i l d
+# ============================================================================ #
+
+# References:
+#
+# https://cloud.google.com/cloud-build/docs/build-config
+#
+# https://cloud.google.com/cloud-build/docs/build-debug-locally
+
+
+# gcloud builds submit --config cloudbuild.yaml .
+#
+# cloud-build-local --config cloudbuild.yml --dryrun=false .
+
+# tars $PWD to bucket called ${PROJECT_ID}_cloudbuild
+
+timeout: 3660s
+
+steps:
+ - name: ubuntu:18.04
+ entrypoint: bash
+ args:
+ - '-c'
+ - |
+ setup/ci_bootstrap.sh &&
+ make build test
+ timeout: 3600s
diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py
new file mode 100755
index 000000000..bbd94dbb3
--- /dev/null
+++ b/cloudera_navigator_tables_used.py
@@ -0,0 +1,467 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-03-09 11:35:47 +0000 (Mon, 09 Mar 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Processes Cloudera Navigator API exported CSV logs to list the tables used (SELECT'ed from)
+
+This allows you to see if you're wasting time maintaining datasets nobody is using
+
+Handles more than naive filtering delimited column numbers which will miss many table and database names:
+
+ 1. table/database name fields are often blank and need to be inferred from SQL queries field
+ (currently limited to 'SELECT ... FROM ...' because JOINs are often complicated by use of table aliases)
+ 2. SQL queries often contain newlines which break the rows up - these are recombined in to single records
+ 3. multi-line SQL queries have comments stripped out to avoid false positives of what is being used
+ 4. where table/database field aren't available, also checks if inferrable from resource field
+ to determine database and table name before parsing SQL which is a last resort
+ 5. optionally ignore selected users by regex
+ - matches user or kerberos principal
+ - eg. to omit ETL service account from skewing data access results
+
+Supports reading directly from gzipped logs if they end in .gz file extension.
+However, the gzip library may have issues around universal newline support, if so, gunzip first.
+
+See cloudera_navigator_audit_logs_download.sh for a script to export these logs
+
+./cloudera_navigator_tables_used.py navigator_audit_2019_hive.csv navigator_audit_2019_impala.csv \\
+ navigator_audit_2020_hive.csv navigator_audit_2020_impala.csv
+
+Output is quoted CSV format to stdout (same as hive_schemas_csv.py for easier comparison):
+
+"database","table"
+
+Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10
+(but may require ongoing tweaks depending on quirks in your data set or changes in the API / logs)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import csv
+import gzip
+import logging
+import os
+import re
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_regex, isInt, isPythonMinVersion
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.3.0'
+
+
+class ClouderaNavigatorTablesUsed(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(ClouderaNavigatorTablesUsed, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ csv.field_size_limit(sys.maxsize)
+ self.delimiter = None
+ self.quotechar = None
+ self.escapechar = None
+ self.timeout_default = None
+ #self.data = {}
+ self.indicies = {}
+ self.len_headers = None
+ self.table_regex = r'[\w\.`]+'
+ self.re_table = re.compile(self.table_regex)
+ # doesn't handle JOINs because SQL pros usually use table aliases
+ self.re_select_from_table = re.compile(r'\bSELECT\b.+\bFROM\b(?:\s|\n)+({table_regex})'\
+ .format(table_regex=self.table_regex), \
+ re.I | re.MULTILINE | re.DOTALL)
+ ignore_statements = [
+ 'SHOW',
+ 'DESCRIBE',
+ 'USE',
+ 'CREATE',
+ 'DROP',
+ 'INSERT',
+ 'DELETE',
+ 'UPDATE',
+ 'GET_TABLES',
+ 'GET_SCHEMAS',
+ 'VIEW_METADATA',
+ 'ANALYZE',
+ r'COMPUTE\s+STATS',
+ 'REFRESH',
+ r'INVALIDATE\s+METADATA',
+ ]
+ self.re_ignore = re.compile(r'^\s*\b(?:' + '|'.join(ignore_statements) + r')\b',\
+ re.I | re.MULTILINE | re.DOTALL)
+ # 2020-01-31T20:45:59.000Z
+ self.re_timestamp = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
+ self.re_ignored_users = None
+ self.csv_writer = None
+ self.operations_to_ignore = [
+ '',
+ 'HIVEREPLICATIONCOMMAND',
+ 'START',
+ 'STOP',
+ 'RESTART',
+ 'LOAD',
+ 'SWITCHDATABASE',
+ 'USE',
+ ]
+
+ def add_options(self):
+ super(ClouderaNavigatorTablesUsed, self).add_options()
+ # must set type to str otherwise csv module gives this error on Python 2.7:
+ # TypeError: "delimiter" must be string, not unicode
+ # type=str worked with argparse but when integrated with CLI then 'from __future__ import unicode_literals'
+ # breaks this - might break in Python 3 if the impyla module doesn't fix behaviour
+ self.add_opt('-d', '--delimiter', default=',', type=str, help='Delimiter to use for outputting (default: ,)')
+ self.add_opt('-Q', '--quotechar', default='"', type=str,
+ help='Generate quoted CSV output (recommended, default is double quote \'"\')')
+ self.add_opt('-E', '--escapechar', help='Escape char if needed (for both reading and writing)')
+ self.add_opt('-i', '--ignore-users', help='Users to ignore, comma separated regex values')
+
+ def process_options(self):
+ super(ClouderaNavigatorTablesUsed, self).process_options()
+ self.delimiter = self.get_opt('delimiter')
+ self.quotechar = self.get_opt('quotechar')
+ self.escapechar = self.get_opt('escapechar')
+ ignore_users = self.get_opt('ignore_users')
+ if ignore_users:
+ ignored_users = ignore_users.split(',')
+ for username in ignored_users:
+ validate_regex(username, 'ignored user')
+ # account for kerberized names - user, user@domain.com or user/host@domain.com
+ self.re_ignored_users = re.compile('^(?:' + '|'.join(ignored_users) + ')(?:[@/]|$)', re.I)
+ if not self.args:
+ self.usage('no CSV file argument given')
+
+ def run(self):
+ quoting = csv.QUOTE_ALL
+ if self.quotechar == '':
+ quoting = csv.QUOTE_NONE
+
+ #fieldnames = ['database', 'table', 'user']
+ fieldnames = ['database', 'table']
+ self.csv_writer = csv.DictWriter(sys.stdout,
+ delimiter=self.delimiter,
+ quotechar=self.quotechar,
+ escapechar=self.escapechar,
+ quoting=quoting,
+ fieldnames=fieldnames)
+ mode = 'rtU'
+ if isPythonMinVersion(3):
+ mode = 'rt'
+ # open(..., encoding="utf-8") is Python 3 only - uses system default otherwise
+ for filename in self.args:
+ if filename.endswith('.gz'):
+ log.debug("processing gzip'd file: %s", filename)
+ with gzip.open(filename, mode) as filehandle:
+ self.process_file(filehandle)
+ else:
+ log.debug("processing file: %s", filename)
+ with open(filename, mode) as filehandle:
+ self.process_file(filehandle)
+
+ #csv_writer.writeheader()
+ #for database in sorted(self.data):
+ # for table in sorted(self.data[database]):
+ # csv_writer.writerow({'database': database,
+ # 'table': table})
+ #if log.isEnabledFor(logging.DEBUG):
+ # sys.stdout.flush()
+
+# Navigator API Audit log output is a mess with duplicate columns and different naming conventions,
+# eg. identical SQL in fields 18 and 36
+# table and database names in fields 19+21 vs 40+41
+# all duplicates with different header names
+#
+# # same result for navigator_audit_2019_impala.csv
+# csv_header_indices.sh navigator_audit_2019_hive.csv
+# 0 Timestamp
+# 1 Username
+# 2 "IP Address"
+# 3 "Service Name"
+# 4 Operation
+# 5 Resource
+# 6 Allowed
+# 7 Impersonator
+# 8 sub_operation
+# 9 entity_id
+# 10 stored_object_name
+# 11 additional_info
+# 12 collection_name
+# 13 solr_version
+# 14 operation_params
+# 15 service
+# 16 operation_text
+# 17 url
+# 18 operation_text
+# 19 table_name
+# 20 resource_path
+# 21 database_name
+# 22 object_type
+# 23 Source
+# 24 Destination
+# 25 Permissions
+# 26 "Delegation Token ID"
+# 27 "Table Name"
+# 28 Family
+# 29 Qualifier
+# 30 "Operation Text"
+# 31 "Database Name"
+# 32 "Table Name"
+# 33 "Object Type"
+# 34 "Resource Path"
+# 35 "Usage Type"
+# 36 "Operation Text"
+# 37 "Query ID"
+# 38 "Session ID"
+# 39 Status
+# 40 "Database Name"
+# 41 "Table Name"
+# 42 "Object Type"
+# 43 Privilege
+
+ def process_file(self, filehandle):
+ csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\')
+ try:
+ # Python 2
+ headers = csv_reader.next()
+ except AttributeError:
+ # Python 3
+ headers = next(csv_reader)
+ self.len_headers = len(headers)
+ # needed to ensure row joining works later on with number of fields left
+ assert self.len_headers == 44
+ user_index = 1
+ operation_index = 4
+ resource_index = 5
+ object_index = 22 # used by collapse_sql_fields to check if SQL was split, do not change to index 33!
+ # --
+ # with massive queries taking the latter 2 is more likely to succeed,
+ # possibly because there is a rare and subtle issue in collapse_sql_fields
+ #table_index = 19
+ #database_index = 21
+ # or
+ table_index = 41
+ database_index = 40
+ # --
+ # fields 18 and 36 are identical SQL - need both to collapse rows later
+ sql_index = 18
+ sql_index2 = 36
+ #assert headers[table_index] == 'table_name' # index 19
+ #assert headers[database_index] == 'database_name' # index 21
+ assert headers[table_index] == 'Table Name' # index 41
+ assert headers[database_index] == 'Database Name' # index 40
+ assert headers[user_index] == 'Username'
+ assert headers[operation_index] == 'Operation'
+ assert headers[resource_index] == 'Resource'
+ assert headers[sql_index] == 'operation_text'
+ assert headers[sql_index2] == 'Operation Text'
+ assert headers[object_index] == 'object_type'
+ self.indicies = {
+ 'user_index': user_index,
+ 'operation_index': operation_index,
+ 'resource_index': resource_index,
+ 'table_index': table_index,
+ 'database_index': database_index,
+ 'sql_index': sql_index,
+ 'sql_index2': sql_index2, # needed for collapsing rows inflated by SQL fragmentation
+ 'object_index': object_index
+ }
+ self.process_rows(csv_reader)
+
+ # logic to reconstruct rows because the Navigator API breaks the record format
+ # with newlines in SQL coming out literally and fragmenting the records
+ def process_rows(self, csv_reader):
+ last_row = []
+ for current_row in csv_reader:
+ #log.debug('current row = %s', current_row)
+ if not current_row:
+ continue
+ if self.is_new_record(current_row):
+ row = last_row
+ last_row = current_row
+ else:
+ last_row += current_row
+ continue
+ if not row:
+ continue
+ self.process_row(row)
+ self.process_row(last_row)
+
+ # originally did this by counting fields but SQL fragmentation generates extra fields
+ def is_new_record(self, current_row):
+ return self.re_timestamp.match(current_row[0])
+
+ def process_row(self, row):
+ if not row:
+ return
+ log.debug('processing row = %s', row)
+ len_row = len(row)
+ log.debug('row len = %s', len_row)
+ if len_row > self.len_headers:
+ row = self.collapse_sql_fields(row=row)
+ len_row = len(row)
+ if len_row != self.len_headers:
+ raise AssertionError('row items ({}) != header items ({}) for offending row: {}'\
+ .format(len_row, self.len_headers, row))
+ (database, table) = self.parse_table(row)
+ self.output(row=row, database=database, table=table)
+
+ def parse_table(self, row):
+ #log.debug(row)
+ user = row[self.indicies['user_index']]
+ # user: 'hari.sekhon'
+ # kerberos principals: 'hari.sekhon@somedomain.com' or 'impala/fqdn@domain.com'
+ if self.re_ignored_users and self.re_ignored_users.match(user):
+ log.debug('skipping row for ignored user %s: %s', user, row)
+ return (None, None)
+ database = row[self.indicies['database_index']].strip()
+ table = row[self.indicies['table_index']].strip()
+ if not database or not table or not self.re_table.match('{}.{}'.format(database, table)):
+ #log.info('table not found in fields for row: %s', row)
+ operation = row[self.indicies['operation_index']]
+ if operation in self.operations_to_ignore:
+ return (None, None)
+ elif operation == 'QUERY':
+ query = row[self.indicies['sql_index']]
+ # cheaper than re_ignore to pre-filter
+ if query in ('GET_TABLES', 'GET_SCHEMAS', 'INVALIDATE METADATA'):
+ return (None, None)
+ (database, table) = self.get_db_table_from_resource(row)
+ if database and table:
+ pass
+ else:
+ log.debug('database/table not found in row: %s', row)
+ log.debug('trying to parse: %s', query)
+ match = self.re_select_from_table.search(query)
+ if match:
+ table = match.group(1)
+ if '.' in table:
+ (database, table) = table.split('.', 1)
+ # could use .search but all these seem to be at beginning
+ elif self.re_ignore.match(query):
+ return (None, None)
+ else:
+ log.warning('failed to parse database/table from query: %s', query)
+ return (None, None)
+ else:
+ log.debug('database/table not found in row and operation is not a query to parse: %s', row)
+ return (None, None)
+ if not table and not database:
+ return (None, None)
+ if table:
+ table = table.lower().strip('`')
+ if ' ' in table:
+ raise AssertionError('table \'{}\' has spaces - parsing error for row: {}'\
+ .format(table, self.index_output(row)))
+ if database:
+ database = database.lower().strip('`')
+ if ' ' in database:
+ raise AssertionError('database \'{}\' has spaces - parsing error for row: {}'\
+ .format(database, self.index_output(row)))
+ if table == 'null':
+ raise AssertionError('table == null - parsing error for row: {}'.format(row))
+ return (database, table)
+
+ def get_db_table_from_resource(self, row):
+ database = None
+ table = None
+ resource = row[self.indicies['resource_index']]
+ if resource and \
+ ':' in resource and \
+ 'null:null' not in resource:
+ # database:table in Resource field
+ (database, table) = resource.split(':', 1)
+ return (database, table)
+
+ def output(self, row, database, table):
+ if not self.re_table.match('{}.{}'.format(database, table)):
+ log.warning('%s.%s does not match table regex', database, table)
+ return
+ # instead of collecting in ram, now just post-process through sort -u
+ # this way it is easier to see live extractions, --debug and correlate
+ #self.data[database] = self.data.get(database, {})
+ #self.data[database][table] = 1
+ if table and not database:
+ log.info('got table but not database for row: %s', row)
+ if database and not table:
+ log.info('got database but not table for row: %s', row)
+ if not table and not database:
+ return
+ #self.csv_writer.writerow({'database': database, 'table': table, 'user': row[self.indicies['user_index']]})
+ self.csv_writer.writerow({'database': database, 'table': table})
+ if log.isEnabledFor(logging.DEBUG):
+ sys.stdout.flush()
+
+ def collapse_sql_fields(self, row):
+ sql_index = self.indicies['sql_index']
+ sql_index2 = self.indicies['sql_index2']
+ object_index = self.indicies['object_index']
+ len_row = len(row)
+ if len_row > self.len_headers:
+ log.debug('collapsing fields in row: %s', row)
+ # divide by 2 to account for this having been done twice in duplicated SQL operational text
+ # Update: appears this broke as only 2nd occurence of SQL operational text field got split to new fields,
+ # which is weird because the log shows both 1st and 2nd SQL text fields were double quoted
+ difference = len_row - self.len_headers
+ # seems first occurrence doesn't get split in some occurence,
+ # wasn't related to open in newline universal mode though
+ # if 2 fields after isn't the /user/hive/warehouse/blah.db then 1st SQL wasn't split
+ # would have to regex /user/hive/warehouse/blah.db(?:/table)?
+ #if not row[sql_index+2].endswith('.db'):
+ # if object field is TABLE or DATABASE then 1st sql field wasn't split
+ if row[object_index] not in ('TABLE', 'DATABASE'):
+ difference /= 2
+ # slice indicies must be integers
+ if not isInt(difference):
+ raise AssertionError("difference in field length '{}' is not an integer for row: {}"\
+ .format(difference, row))
+ difference = int(difference)
+ row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]])
+ row = row[:sql_index] + row[sql_index+difference:]
+ row[sql_index2] = ','.join([self.sql_decomment(_) for _ in row[sql_index2:difference]])
+ row = row[:sql_index2] + row[sql_index2+difference:]
+ log.debug('collapsed row: %s', row)
+ else:
+ log.debug('not collapsing row: %s', row)
+ return row
+
+ @staticmethod
+ def sql_decomment(string):
+ return string.split('--')[0].strip()
+
+ @staticmethod
+ def index_output(obj):
+ return '\n'.join(['{}\t{}'.format(index, item) for (index, item) in enumerate(obj)])
+
+
+if __name__ == '__main__':
+ ClouderaNavigatorTablesUsed().main()
diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py
new file mode 100755
index 000000000..972716236
--- /dev/null
+++ b/cloudera_navigator_tables_used_postgres.py
@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-03-16 19:21:24 +0000 (Mon, 16 Mar 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Processes Cloudera Navigator CSV logs exported from PostgreSQL to list the tables used (SELECT'ed from)
+
+This allows you to see if you're wasting time maintaining datasets nobody is using
+
+See cloudera_navigator_audit_logs_export_postgres.sh for a script to export these logs
+
+Supports reading directly from gzipped logs if they end in .gz file extension.
+However, the gzip library may have issues around universal newline support, if so, gunzip first.
+
+./cloudera_navigator_tables_used_postgres.py nav.public.hive_audit_events_*.csv.gz \\
+ nav.public.impala_audit_events_*.csv.gz ...
+
+Output is quoted CSV format to stdout (same as hive_schemas_csv.py for easier comparison):
+
+"database","table"
+
+Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import csv
+#import logging
+import os
+import re
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, isInt
+ #from harisekhon import CLI
+ from cloudera_navigator_tables_used import ClouderaNavigatorTablesUsed
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.2.1'
+
+
+class ClouderaNavigatorTablesUsedPostgreSQL(ClouderaNavigatorTablesUsed):
+
+ def __init__(self):
+ # Python 2.x
+ super(ClouderaNavigatorTablesUsedPostgreSQL, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ # recombine records due to SQL \n breaking up records, new records start like:
+ # 306529,1574163624392,1,hive,
+ self.re_new_record = re.compile(r'^\d+,\d+,[01],(?:hive|impala),')
+ # get db + table from resource path (just one layer of checks)
+ self.re_resource = re.compile(r'/(\w+)\.db/(\w+)')
+
+# Navigator table logs:
+#
+# gzcat nav.public.hive_audit_events_2019_11_19.csv.gz | csv_header_indices.sh
+# 0 id
+# 1 event_time
+# 2 allowed
+# 3 service_name
+# 4 username
+# 5 ip_addr
+# 6 operation
+# 7 database_name
+# 8 object_type
+# 9 table_name
+# 10 operation_text
+# 11 impersonator
+# 12 resource_path
+# 13 object_usage_type
+
+# gzcat nav.public.impala_audit_events_2019_11_19.csv.gz | csv_header_indices.sh
+# 0 id
+# 1 event_time
+# 2 allowed
+# 3 service_name
+# 4 username
+# 5 impersonator
+# 6 ip_addr
+# 7 operation
+# 8 query_id
+# 9 session_id
+# 10 status
+# 11 database_name
+# 12 object_type
+# 13 table_name
+# 14 privilege
+# 15 operation_text
+
+ def process_file(self, filehandle):
+ csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\')
+ headers = csv_reader.next()
+ self.len_headers = len(headers)
+ # needed to ensure row joining works later on with number of fields left
+ assert self.len_headers == 14 or self.len_headers == 16
+ user_index = 4
+ assert headers[user_index] == 'username'
+ # Hive postgres audit log
+ if self.len_headers == 14:
+ operation_index = 6
+ database_index = 7
+ table_index = 9
+ sql_index = 10
+ resource_index = 12
+ assert headers[resource_index] == 'resource_path'
+ # Impala postgres audit log
+ elif self.len_headers == 16:
+ operation_index = 7
+ database_index = 11
+ table_index = 13
+ sql_index = 15
+ resource_index = None
+ else:
+ raise AssertionError('headers != 14 or 16 - unrecognized audit log - not Hive or Impala')
+ assert headers[sql_index] == 'operation_text'
+ assert headers[database_index] == 'database_name'
+ assert headers[table_index] == 'table_name'
+ assert headers[operation_index] == 'operation'
+ self.indicies = {
+ 'user_index': user_index,
+ 'operation_index': operation_index,
+ 'resource_index': resource_index,
+ 'table_index': table_index,
+ 'database_index': database_index,
+ 'sql_index': sql_index,
+ }
+ self.process_rows(csv_reader)
+
+ def is_new_record(self, current_row):
+ return self.re_new_record.match(','.join(current_row))
+
+ def parse_table(self, row):
+ #log.debug(row)
+ user = row[self.indicies['user_index']]
+ # user: 'hari.sekhon'
+ # kerberos principals: 'hari.sekhon@somedomain.com' or 'impala/fqdn@domain.com'
+ if self.re_ignored_users and self.re_ignored_users.match(user):
+ log.debug('skipping row for ignored user %s: %s', user, row)
+ return (None, None)
+ database = row[self.indicies['database_index']].strip()
+ table = row[self.indicies['table_index']].strip()
+ if not database or not table or not self.re_table.match('{}.{}'.format(database, table)):
+ #log.info('table not found in fields for row: %s', row)
+ operation = row[self.indicies['operation_index']]
+ if operation in self.operations_to_ignore:
+ return (None, None)
+ elif operation == 'QUERY':
+ query = row[self.indicies['sql_index']]
+ # cheaper than re_ignore to pre-filter
+ if query in ('GET_TABLES', 'GET_SCHEMAS', 'INVALIDATE METADATA'):
+ return (None, None)
+ (database, table) = self.get_db_table_from_resource(row)
+ if database and table:
+ pass
+ else:
+ log.debug('database/table not found in row: %s', row)
+ log.debug('trying to parse: %s', query)
+ match = self.re_select_from_table.search(query)
+ if match:
+ table = match.group(1)
+ if '.' in table:
+ (database, table) = table.split('.', 1)
+ # could use .search but all these seem to be at beginning
+ elif self.re_ignore.match(query):
+ return (None, None)
+ else:
+ log.warning('failed to parse database/table from query: %s', query)
+ return (None, None)
+ else:
+ log.debug('database/table not found in row and operation is not a query to parse: %s', row)
+ return (None, None)
+ if not table and not database:
+ return (None, None)
+ if table:
+ table = table.lower().strip('`')
+ if ' ' in table:
+ raise AssertionError('table \'{}\' has spaces - parsing error for row: {}'\
+ .format(table, self.index_output(row)))
+ if database:
+ database = database.lower().strip('`')
+ if ' ' in database:
+ raise AssertionError('database \'{}\' has spaces - parsing error for row: {}'\
+ .format(database, self.index_output(row)))
+ if table == 'null':
+ raise AssertionError('table == null - parsing error for row: {}'.format(row))
+ return (database, table)
+
+ def get_db_table_from_resource(self, row):
+ # only available for hive audit logs, not impala
+ if self.indicies['resource_index'] is None:
+ return (None, None)
+ database = None
+ table = None
+ resource = row[self.indicies['resource_index']]
+ if resource:
+ match = self.re_resource.search(resource)
+ if match:
+ database = match.group(1)
+ table = match.group(2)
+ return (database, table)
+
+ def collapse_sql_fields(self, row):
+ sql_index = self.indicies['sql_index']
+ len_row = len(row)
+ if len_row > self.len_headers:
+ log.debug('collapsing fields in row: %s', row)
+ difference = len_row - self.len_headers
+ # slice indicies must be integers
+ if not isInt(difference):
+ raise AssertionError("difference in field length '{}' is not an integer for row: {}"\
+ .format(difference, row))
+ difference = int(difference)
+ row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]])
+ row = row[:sql_index] + row[sql_index+difference:]
+ log.debug('collapsed row: %s', row)
+ else:
+ log.debug('not collapsing row: %s', row)
+ return row
+
+
+if __name__ == '__main__':
+ ClouderaNavigatorTablesUsedPostgreSQL().main()
diff --git a/codefresh.yml b/codefresh.yml
new file mode 100644
index 000000000..7cddcaf7c
--- /dev/null
+++ b/codefresh.yml
@@ -0,0 +1,43 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-24 17:43:07 +0000 (Mon, 24 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C o d e f r e s h C I
+# ============================================================================ #
+
+# https://codefresh.io/docs/docs/codefresh-yaml/
+
+version: "1.0"
+stages:
+ - "checkout"
+ - "build"
+steps:
+ checkout:
+ type: "git-clone"
+ description: "Cloning main repository..."
+ repo: '${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}'
+ revision: "${{CF_REVISION}}"
+ stage: "checkout"
+ build:
+ title: Running docker image
+ type: freestyle
+ working_directory: '${{CF_REPO_NAME}}'
+ arguments:
+ image: 'ubuntu:18.04'
+ commands:
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci
+ - make test
diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py
index 34069c12b..2e675c253 100755
--- a/crunch_accounting_csv_statement_converter.py
+++ b/crunch_accounting_csv_statement_converter.py
@@ -1,18 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# coding=utf-8
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-02-04 23:24:30 +0000 (Mon, 04 Feb 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -61,7 +61,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.6.2'
+__version__ = '0.7.4'
class CrunchAccountingCsvStatementConverter(CLI):
@@ -107,6 +107,7 @@ def run(self):
log.info("converted '%s' => '%s'", filename, target_filename)
else:
log.error("FAILED to convert filename '%s'", filename)
+ sys.exit(2)
log.info('Final Balance: {}'.format(self.running_balance))
def convert(self, filename, target_filename):
@@ -157,7 +158,7 @@ def reverse_contents(filename):
return tmp_filename
def detect_columns(self, csvreader):
- headers = csvreader.next()
+ headers = next(csvreader)
if headers[0][0] == '{':
log.error('JSON opening braces detected, not a CSV?')
return False
@@ -167,16 +168,21 @@ def detect_columns(self, csvreader):
# want Transaction Date and not Posted Date
if 'Date' in value and not 'Posted' in value:
positions['date'] = position
- elif 'Merchant Name' in value:
- positions['desc'] = position
- elif 'Amount' in value:
+ # Original Amount column will be original currency eg 499 USD, but we only want native currency eg. 421.33
+ elif 'Amount' in value and not 'Original' in value:
positions['amount'] = position
elif 'Balance' in value:
balance_position = position
+ # Barclaycard CSVs
+ elif 'Merchant Name' in value:
+ positions['desc'] = position
+ # Barclays CSVs
+ elif 'Memo' in value:
+ positions['desc'] = position
for pos in positions:
if positions[pos] is None:
log.error('field %s not found', pos)
- return False
+ sys.exit(1)
if balance_position is None and self.running_balance is None:
self.usage('no balance column detected, please specify --starting-balance')
return (positions, balance_position)
@@ -197,7 +203,8 @@ def get_csvreader(filename):
csvreader = csv.reader(filehandle, dialect)
except csv.Error as _:
log.warning('file %s: %s', filename, _)
- return None
+ # in Python 2 must be string not unicode
+ csvreader = csv.reader(filehandle, delimiter=str(','), quotechar=None)
csvreader = CrunchAccountingCsvStatementConverter.validate_csvreader(csvreader, filename)
filehandle.seek(0)
return csvreader
@@ -219,11 +226,12 @@ def validate_csvreader(csvreader, filename):
# extra protection along the same lines as anti-json:
# the first char of field should be alphanumeric, not syntax
# however instead of isAlnum allow quotes for quoted CSVs to pass validation
- if not isChars(field_list[0][0], 'A-Za-z0-9"'):
- log.error('non-alphanumeric / quote opening character detected in CSV')
- return None
+ if field_list[0] not in ("", " ") and not isChars(field_list[0][0], 'A-Za-z0-9"'):
+ log.warning('non-alphanumeric / quote opening character detected in CSV first field' + \
+ '"{}"'.format(field_list[0]))
+ #return None
count += 1
- except csv.Error as _:
+ except csv.Error as _:
log.warning('file %s, line %s: %s', filename, csvreader.line_num, _)
return None
if count == 0:
diff --git a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh
new file mode 100755
index 000000000..61a1235bd
--- /dev/null
+++ b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# shellcheck disable=SC1090
+. "$srcdir/lib.sh"
+
+# statements should be named in format: Barclaycard_Statement_YYYY-MM-DD.csv
+export STATEMENT_GLOB="Barclaycard_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv"
+
+generate_crunch_statements --credit-card --reverse-order
diff --git a/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh
new file mode 100755
index 000000000..f708fec0c
--- /dev/null
+++ b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# shellcheck disable=SC1090
+. "$srcdir/lib.sh"
+
+# statements should be named in format: Barclays_Statement_YYYY-MM-DD.csv
+export STATEMENT_GLOB="Barclays_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv"
+
+# Barclays CSV statements often have whitespace starting fields instead of blank or 'null'
+# unfortunately this resets all the original CSV timestamps each run so it's best to do only where needed
+# UPDATE: no longer necessary, converter just ignores these blank fields now in the validation
+#for statement in $STATEMENT_GLOB; do
+ #perl -pi -e 's/^\s+,/,/' "$statement"
+#done
+
+generate_crunch_statements --reverse-order
diff --git a/crunch_accounting_csv_statement_converter_scripts/lib.sh b/crunch_accounting_csv_statement_converter_scripts/lib.sh
new file mode 100755
index 000000000..e77e66834
--- /dev/null
+++ b/crunch_accounting_csv_statement_converter_scripts/lib.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# shellcheck disable=SC1090
+#. "$srcdir/lib/utils.sh"
+
+converter="$srcdir/../crunch_accounting_csv_statement_converter.py"
+
+get_latest_crunch_statement(){
+ local latest_crunch_statement
+ for statement in $STATEMENT_GLOB; do
+ crunch_statement="${statement%.csv}_crunch.csv"
+ if [ -f "$crunch_statement" ]; then
+ latest_crunch_statement="$crunch_statement"
+ fi
+ done
+ if [ -n "${latest_crunch_statement:-}" ]; then
+ echo "$latest_crunch_statement"
+ fi
+}
+
+get_final_balance_from_statement(){
+ local crunch_statement="$1"
+ if ! [[ "$crunch_statement" =~ _crunch.csv$ ]]; then
+ echo "invalid statement passed to get_final_balance_from_statement(), must be *_crunch.csv" >&2
+ exit 1
+ fi
+ tail -n 1 "$crunch_statement" | awk -F, '{print $4}'
+}
+
+get_starting_balance(){
+ local starting_balance
+ local latest_crunch_statement="$1"
+ if [ -n "${latest_crunch_statement:-}" ]; then
+ echo "latest crunch statement is $latest_crunch_statement" >&2
+ starting_balance="$(get_final_balance_from_statement "$latest_crunch_statement")"
+ else
+ echo "no latest crunch statement, getting starting balance from environment variable \$STARTING_BALANCE" >&2
+ starting_balance="${STARTING_BALANCE:-}"
+ if [ -z "$starting_balance" ]; then
+ echo "last crunch statement not found, you must specify the last balance manually via the environment variable \$STARTING_BALANCE" >&2
+ exit 1
+ fi
+ fi
+ echo "starting balance: $starting_balance" >&2
+ echo "$starting_balance"
+}
+
+generate_crunch_statements(){
+ # only generate statements newer than the last generated one which provides the starting balance
+ local passed_latest_statement=0
+ local latest_crunch_statement
+ local starting_balance
+ latest_crunch_statement="$(get_latest_crunch_statement)"
+ if [ -z "$latest_crunch_statement" ]; then
+ passed_latest_statement=1
+ fi
+ starting_balance="$(get_starting_balance "$latest_crunch_statement")"
+ for statement in $STATEMENT_GLOB; do
+ crunch_statement="${statement%.csv}_crunch.csv"
+ if [ -f "$crunch_statement" ]; then
+ if [ $passed_latest_statement = 0 ] &&
+ [ "$crunch_statement" = "$latest_crunch_statement" ]; then
+ passed_latest_statement=1
+ fi
+ continue
+ fi
+ if [ $passed_latest_statement -lt 1 ]; then
+ continue
+ fi
+ "$converter" "$@" --starting-balance "$starting_balance" "$statement"
+ starting_balance="$(get_final_balance_from_statement "$crunch_statement")"
+ done
+}
diff --git a/docker_pull_all_images.sh b/docker_pull_all_images.sh
index 430ac70d5..e6d39fd0d 100755
--- a/docker_pull_all_images.sh
+++ b/docker_pull_all_images.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/docker_pull_all_images_all_tags.sh b/docker_pull_all_images_all_tags.sh
index 3656395cb..d12c3e25b 100755
--- a/docker_pull_all_images_all_tags.sh
+++ b/docker_pull_all_images_all_tags.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/docker_pull_all_tags.sh b/docker_pull_all_tags.sh
index e555af230..229f3fe68 100755
--- a/docker_pull_all_tags.sh
+++ b/docker_pull_all_tags.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/docker_registry_show_tags.py b/docker_registry_show_tags.py
index 50d1af743..8c18d3cdc 100755
--- a/docker_registry_show_tags.py
+++ b/docker_registry_show_tags.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-10 11:26:49 +0100 (Tue, 10 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/dockerfiles_check_git_branches.py b/dockerfiles_check_git_branches.py
index a7edf5195..662dce3e2 100755
--- a/dockerfiles_check_git_branches.py
+++ b/dockerfiles_check_git_branches.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-20 20:24:12 +0100 (Fri, 20 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
r"""
@@ -91,7 +91,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.7.2'
+__version__ = '0.7.3'
class DockerfileGitBranchCheckTool(CLI):
@@ -321,8 +321,8 @@ def check_file(self, filename, branch):
def check_dockerfile_arg(self, filename, branch):
log.debug('check_dockerfile_arg({0}, {1})'.format(filename, branch))
- branch_base = str(branch).replace('-dev', '')
- (branch_base, branch_versions) = self.branch_version(branch)
+ branch_stripped = str(branch).replace('-dev', '')
+ (branch_base, branch_versions) = self.branch_version(branch_stripped)
with open(filename) as filehandle:
version_index = 0
for line in filehandle:
diff --git a/dockerfiles_check_git_tags.py b/dockerfiles_check_git_tags.py
index 996f787b8..3e0b627bd 100755
--- a/dockerfiles_check_git_tags.py
+++ b/dockerfiles_check_git_tags.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-20 20:24:12 +0100 (Fri, 20 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
r"""
diff --git a/dockerhub_search.py b/dockerhub_search.py
index cb226e59f..ba7034145 100755
--- a/dockerhub_search.py
+++ b/dockerhub_search.py
@@ -1,17 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
+# args: harisekhon
#
# Author: Hari Sekhon
# Date: 2016-05-27 13:15:30 +0100 (Fri, 27 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -20,12 +21,20 @@
Mimics 'docker search' results format but more flexible
-Docker CLI doesn't currently support configuring the returned number of search results and always returns 25:
+Older Docker CLI didn't support configuring the returned number of search results and always returned 25:
https://github.com/docker/docker/issues/23055
Verbose mode will also show a summary for number of results displayed and total number of results available
+Caveat: maxes out at 100 results, to iterate for more than that see dockerhub_search.sh
+
+See also:
+
+ dockerhub_search.sh
+
+in the DevOps Bash tools repo - https://github.com/HariSekhon/DevOps-Python-tools
+
"""
from __future__ import absolute_import
@@ -56,7 +65,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.6'
+__version__ = '0.6.2'
class DockerHubSearch(CLI):
@@ -71,7 +80,7 @@ def __init__(self):
self.quiet = False
def add_options(self):
- self.add_opt('-n', '--num', '--limit', default=50,
+ self.add_opt('-l', '--limit', default=50, type=int,
help='Number of results to return (default: 50)')
self.add_opt('-q', '--quiet', action='store_true',
help='Output only the image names, one per line (useful for shell scripting)')
@@ -84,9 +93,9 @@ def run(self):
self.quiet = self.get_opt('quiet')
term = self.args[0]
log.info('term: %s', term)
- num = self.get_opt('num')
- validate_int(num, 'limit', 1, 1000)
- self.print_results(self.args[0], num)
+ limit = self.get_opt('limit')
+ validate_int(limit, 'limit', 1, 1000)
+ self.print_results(self.args[0], limit)
def print_results(self, term, limit=None):
data = self.search(term, limit)
diff --git a/dockerhub_show_tags.py b/dockerhub_show_tags.py
index 88c9ddc08..59cd6cb5f 100755
--- a/dockerhub_show_tags.py
+++ b/dockerhub_show_tags.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-10 11:26:49 +0100 (Tue, 10 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -32,6 +32,7 @@
import json
import logging
import os
+import re
import sys
import traceback
import urllib
@@ -52,7 +53,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.6.2'
+__version__ = '0.6.3'
class DockerHubTags(CLI):
@@ -79,8 +80,11 @@ def run(self):
self.usage('no repos given as args')
self.quiet = self.get_opt('quiet')
if not self.quiet:
+ # cheaper but lgtm hassling me, not a security issue but will shut them up
print('\nDocker', end='')
- if 'registry.hub.docker.com' in self.url_base:
+ #if 'registry.hub.docker.com' in self.url_base:
+ # match anchors but I prefer explicit anchor, more intuitive for other generic language coders
+ if re.match(r'^https://registry\.hub\.docker\.com/', self.url_base):
print('Hub')
else:
print(' Registry: {0}'.format(self.url_base.split('/v2', 1)[0]))
diff --git a/find_active_apache_drill.py b/find_active_apache_drill.py
index ff4461960..1e89fb676 100755
--- a/find_active_apache_drill.py
+++ b/find_active_apache_drill.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_apache_drill2.py b/find_active_apache_drill2.py
index 4dcdf8219..364b0eda3 100755
--- a/find_active_apache_drill2.py
+++ b/find_active_apache_drill2.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_cassandra.py b/find_active_cassandra.py
index 67360ccc3..c16ee217c 100755
--- a/find_active_cassandra.py
+++ b/find_active_cassandra.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_elasticsearch.py b/find_active_elasticsearch.py
index 42d98d4b8..be449bc79 100755
--- a/find_active_elasticsearch.py
+++ b/find_active_elasticsearch.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Tue Sep 5 10:49:49 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hadoop2_namenode.py b/find_active_hadoop2_namenode.py
index bdb7ee5e4..7aad7e77d 100755
--- a/find_active_hadoop2_namenode.py
+++ b/find_active_hadoop2_namenode.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Tue Sep 5 10:49:49 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hadoop_namenode.py b/find_active_hadoop_namenode.py
index ff578d9ef..1f6f2d44d 100755
--- a/find_active_hadoop_namenode.py
+++ b/find_active_hadoop_namenode.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Tue Sep 5 10:49:49 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hadoop_yarn_resource_manager.py b/find_active_hadoop_yarn_resource_manager.py
index d1179c7cc..f2f0e48e3 100755
--- a/find_active_hadoop_yarn_resource_manager.py
+++ b/find_active_hadoop_yarn_resource_manager.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 6 14:44:38 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hbase_master.py b/find_active_hbase_master.py
index e89c87613..cc7de0222 100755
--- a/find_active_hbase_master.py
+++ b/find_active_hbase_master.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hbase_stargate.py b/find_active_hbase_stargate.py
index c62b1756c..79dc122fa 100755
--- a/find_active_hbase_stargate.py
+++ b/find_active_hbase_stargate.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hbase_thrift.py b/find_active_hbase_thrift.py
index a83594730..06e44a414 100755
--- a/find_active_hbase_thrift.py
+++ b/find_active_hbase_thrift.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_impala.py b/find_active_impala.py
index cce6eb399..ae086f46a 100755
--- a/find_active_impala.py
+++ b/find_active_impala.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_impala_catalog.py b/find_active_impala_catalog.py
index 7fc7a143e..cb6ea0de3 100755
--- a/find_active_impala_catalog.py
+++ b/find_active_impala_catalog.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_impala_statestore.py b/find_active_impala_statestore.py
index 3ec124494..049ed78f5 100755
--- a/find_active_impala_statestore.py
+++ b/find_active_impala_statestore.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_kubernetes_api.py b/find_active_kubernetes_api.py
index 0ef8165f8..a93b0bc81 100755
--- a/find_active_kubernetes_api.py
+++ b/find_active_kubernetes_api.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_oozie.py b/find_active_oozie.py
index 21b9271d6..156bb51a9 100755
--- a/find_active_oozie.py
+++ b/find_active_oozie.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_presto_coordinator.py b/find_active_presto_coordinator.py
index 13c8f6546..9b0050a47 100755
--- a/find_active_presto_coordinator.py
+++ b/find_active_presto_coordinator.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_server.py b/find_active_server.py
index edcf0fb49..30810a448 100755
--- a/find_active_server.py
+++ b/find_active_server.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-09-29 15:00:36 +0100 (Thu, 29 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
r"""
@@ -96,7 +96,7 @@
See also Advanced HAProxy configurations (part of the Advanced Nagios Plugins Collection) at:
- https://github.com/harisekhon/haproxy-configs
+ https://github.com/HariSekhon/HAProxy-configs
"""
@@ -112,14 +112,20 @@
import subprocess
import sys
#from threading import Thread
-from multiprocessing.pool import ThreadPool, cpu_count
+from multiprocessing.pool import ThreadPool
+from multiprocessing import cpu_count
# prefer blocking semantics of que.get() rather than handling deque.popleft() => 'IndexError: pop from an empty deque'
#from collections import deque
-import Queue
import traceback
from random import shuffle
+# Python 2 Queue vs Python 3 queue module :-/
+if sys.version[0] == '2':
+ import Queue as queue # pylint: disable=import-error
+else:
+ import queue as queue # pylint: disable=import-error
try:
- import requests
+ # false positive from pylint, queue is imported first
+ import requests # pylint: disable=wrong-import-order
except ImportError:
print(traceback.format_exc(), end='')
sys.exit(4)
@@ -137,7 +143,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.8.4'
+__version__ = '0.8.6'
class FindActiveServer(CLI):
@@ -156,7 +162,7 @@ def __init__(self):
self.request_timeout = None
self.default_num_threads = min(cpu_count() * 4, 100)
self.num_threads = None
- self.queue = Queue.Queue()
+ self.queue = queue.Queue()
self.pool = None
def add_options(self):
diff --git a/find_active_solrcloud.py b/find_active_solrcloud.py
index b6683fafb..659eceb59 100755
--- a/find_active_solrcloud.py
+++ b/find_active_solrcloud.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Tue Sep 5 10:49:49 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_duplicate_files.py b/find_duplicate_files.py
index 00b45fe2e..4c452b03f 100755
--- a/find_duplicate_files.py
+++ b/find_duplicate_files.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-08-14 09:50:03 +0100 (Sun, 14 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -33,12 +33,17 @@
4. regex capture matching portion - specify a regex to match against the filenames with capture (brackets) and the
captured portion will be compared among files. If no capture brackets are detected
then will treat the entire regex as the capture.
- Regex is case insensitive by default
+ Regex is case insensitive by default and applies only to the file's basename
+
+Exits with exit code 4 if duplicates are found
Can restrict methods of finding duplicates to any combination of --name / --size / --checksum (checksum implies size as
an efficiency shortcut) / --regex. If none are specified then will try name, size + checksum. If specifying any one of
these options then the others will not run unless also explicitly specified.
+If you want to find files that are probably the same by byte count but may not have the same checksum due to minor
+corruption, such as large media files, then specify --size but do not specify --checksum which supercedes it
+
Caveats:
- The limitation of the checksum approach is that it can't determine files as duplicates if there is any
@@ -47,10 +52,12 @@
- By default this program will short-circuit to stop processing a file as soon as it is determined to be a duplicate
file via one of the above methods in that order for efficiency. This means that if 2 files have duplicate names,
and a third has a different name but the same checksum as the second one, the second one's size + checksum will not have
-been recorded stored and so the third duplicate will not be detected. However, if you removed one duplicate the next
-run of this program would find the other duplicate via the other dimension of checking. Given it's a rare condition
-it's probably not worth the extra overhead in everyday use but this behaviour can be overridden by specifying the
---no-short-circuit option too run every check on every file. Be aware this will slow down the process.
+been checked and so a third duplicate with a different name will not be detected by size / checksum. In most cases this
+is a good thing to finish quicker and avoid unnecessary checksumming which is computationally expensive and time
+consuming for large files. If you remove one duplicate then the next run of this program would find the other
+duplicate via the additional checks of size and checksumming. Given it's a rare condition it's probably not worth the
+extra overhead in everyday use but this behaviour can be overridden by specifying the --no-short-circuit option to run
+every check on every file. Be aware this will slow down the process.
To see progress of which files are matching size, backtracking to hash them for comparison etc
use --verbose twice or -vv. To see which files are being checked use triple verbose mode -vvv
@@ -81,7 +88,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.5.4'
+__version__ = '0.6.2'
class FindDuplicateFiles(CLI):
@@ -165,7 +172,7 @@ def process_args(self):
log_option('compare by name', self.compare_by_name)
log_option('compare by size', self.compare_by_size)
log_option('compare by checksum', self.compare_by_checksum)
- log_option('compare by regex', True if self.regex else False)
+ log_option('compare by regex', bool(self.regex))
return args
@staticmethod
@@ -210,36 +217,36 @@ def run(self):
for filepath in sorted(self.dup_filepaths):
print(filepath)
sys.exit(4)
- print('Duplicates detected!\n')
+ print('# Duplicates detected!')
if self.dups_by_name:
- print('Duplicates by name:\n')
+ print('\n# Duplicates by name:\n')
for basename in self.dups_by_name:
- print("--\nbasename '{0}':".format(basename))
+ print("# --\n# basename '{0}':".format(basename))
for filepath in sorted(self.dups_by_name[basename]):
print(filepath)
if self.dups_by_size:
- print('Duplicates by size:\n')
+ print('\n# Duplicates by size:\n')
for size in self.dups_by_size:
- print("--\nsize '{0}' bytes:".format(size))
+ print("# --\n# size '{0}' bytes:".format(size))
for filepath in sorted(self.dups_by_size[size]):
print(filepath)
if self.dups_by_hash:
- print('Duplicates by checksum:\n')
+ print('\n# Duplicates by checksum:\n')
for checksum in self.dups_by_hash:
- print("--\nchecksum '{0}':".format(checksum))
+ print("# --\n# checksum '{0}':".format(checksum))
for filepath in sorted(self.dups_by_hash[checksum]):
print(filepath)
if self.dups_by_regex:
- print('Duplicates by regex match ({0}):\n'.format(self.regex))
+ print('\n# Duplicates by regex match ({0}):\n'.format(self.regex))
for matching_portion in self.dups_by_regex:
- print("--\nregex matching portion '{0}':".format(matching_portion))
+ print("# --\n# regex matching portion '{0}':".format(matching_portion))
for filepath in sorted(self.dups_by_regex[matching_portion]):
print(filepath)
sys.exit(4)
elif self.failed:
sys.exit(2)
else:
- print('No Duplicates Found')
+ print('# No Duplicates Found')
sys.exit(0)
# def check_path(self, path):
@@ -297,6 +304,7 @@ def check_path(self, path):
def is_file_dup(self, filepath):
log.debug("checking file path '%s'", filepath)
+ # pylint: disable=no-else-return
if os.path.islink(filepath):
log.debug("ignoring symlink '%s'", filepath)
return False
@@ -340,8 +348,7 @@ def is_file_dup_by_name(self, filepath):
self.dups_by_name[basename].add(self.files[basename])
self.dups_by_name[basename].add(filepath)
return True
- else:
- self.files[basename] = filepath
+ self.files[basename] = filepath
return False
def is_file_dup_by_size(self, filepath):
@@ -397,20 +404,21 @@ def is_file_dup_by_hash(self, filepath):
return False
def is_file_dup_by_regex(self, filepath):
- match = re.search(self.regex, filepath)
+ #match = re.search(self.regex, filepath)
+ basename = os.path.basename(filepath)
+ match = re.search(self.regex, basename)
if match:
log.debug("regex matched file '%s'", filepath)
if match.groups():
capture = match.group(1)
- if capture in self.regex_captures:
- self.dups_by_regex[capture] = self.dups_by_regex.get(capture, set())
- self.dups_by_regex[capture].add(self.regex_captures[capture])
- self.dups_by_regex[capture].add(filepath)
- return True
- else:
- self.regex_captures[capture] = filepath
else:
- log.error('no capture detected! Did you forget to specify the (brackets) to capture in the regex?')
+ capture = match.group(0)
+ if capture in self.regex_captures:
+ self.dups_by_regex[capture] = self.dups_by_regex.get(capture, set())
+ self.dups_by_regex[capture].add(self.regex_captures[capture])
+ self.dups_by_regex[capture].add(filepath)
+ return True
+ self.regex_captures[capture] = filepath
return False
diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py
new file mode 100755
index 000000000..f982cbddd
--- /dev/null
+++ b/find_missing_files_in_sequence.py
@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-07-31 11:03:17 +0100 (Fri, 31 Jul 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Finds missing files by numeric sequence, assuming a uniformly numbered file naming convention across files
+
+Files / directories are given as arguments or via standard input
+
+Directories are recursed and their files examined for missing numbers before each one
+
+Only supply files / directories that should be sharing a contiguously numbered file naming convention in each
+single run of this tool
+
+Accounts for zero padding in numbered files
+
+Caveats:
+
+- This is more complicated than you'd first think as there are so many file naming variations that no code could ever
+ be universally bulletproof and will likely require advanced regex tuning to match your use case and naming convention
+
+- Won't detect missing files higher than the highest numbered file as there is no way to know how many there should be.
+ If you are looking for missing MP3 files, then you might be able to check the mp3 tag metadata using programs like
+ 'mediainfo' to get the total number of tracks and see if the files go that high
+
+- Returns globs by default instead of explicit missing filenames since suffixes can vary after numbers. If you have a
+ simple enough use case with a single fixed filename convention such as 'blah_01.txt' then you can find code to print
+ the missing files more explicitly, but in the general case you cannot account for suffix naming that isn't consistent,
+ such as chapters of audiobooks eg.
+
+ 'blah 01 - chapter about X.mp3'
+ 'blah 02 - chapter about Y.mp3'
+
+ so in the general case you cannot always infer suffixes, hence why it is left as globs. If you are sure that the
+ suffixes don't change then you can specify --fixed-suffix and it will infer each file's suffix as the basis for any
+ numerically missing files in the sequence, but if used where this is not the case, it'll generate a lot of false
+ positives that the default globbing mode would have handled
+
+- Doesn't currently find entire missing CD / disks in the naming format, but you should be able to see those cases
+ easily by eye
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import glob
+#import logging
+import os
+import re
+import sys
+import traceback
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ from harisekhon.utils import log, log_option, validate_regex, isInt, UnknownError
+ from harisekhon import CLI
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.3.2'
+
+
+# pylint: disable=too-many-instance-attributes
+class FindMissingFiles(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(FindMissingFiles, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.paths = []
+ self.regex_default = r'(? 1:
+ file_number = self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix)
+ if self.missing_files:
+ print('\n'.join(reversed(self.missing_files)))
+ self.missing_files = []
+
+ def determine_missing_file_backfill(self, file_prefix, file_number, padding, file_suffix):
+ file_number -= 1
+ if self.fixed_suffix:
+ explicit_last_filename = '{}{:0>%(padding)s}{}' % {'padding': padding}
+ explicit_last_filename = explicit_last_filename.format(file_prefix, file_number, file_suffix)
+ if not os.path.isfile(explicit_last_filename):
+ self.missing_files.append(explicit_last_filename)
+ else:
+ file_number = -1
+ else:
+ expected_last_filename_glob = '{}{:0>%(padding)s}*' % locals()
+ expected_last_filename_glob = expected_last_filename_glob.format(file_prefix, file_number)
+ if not glob.glob(expected_last_filename_glob):
+ self.missing_files.append(expected_last_filename_glob)
+ else:
+ file_number = -1
+ return file_number
+
+
+if __name__ == '__main__':
+ FindMissingFiles().main()
diff --git a/gcp_cloud_function_ifconfig/.gcloudignore b/gcp_cloud_function_ifconfig/.gcloudignore
new file mode 100644
index 000000000..42de3fd00
--- /dev/null
+++ b/gcp_cloud_function_ifconfig/.gcloudignore
@@ -0,0 +1,22 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020)
+#
+# vim:ts=4:sts=4:sw=4:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore
+
+.git
+.gcloudignore
+deploy.sh
+test/
+tests/
diff --git a/gcp_cloud_function_ifconfig/Makefile b/gcp_cloud_function_ifconfig/Makefile
new file mode 100644
index 000000000..656b78879
--- /dev/null
+++ b/gcp_cloud_function_ifconfig/Makefile
@@ -0,0 +1,22 @@
+#
+# Author: Hari Sekhon
+# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021)
+#
+# vim:ts=4:sts=4:sw=4:noet
+#
+# https://github.com/HariSekhon/Kubernetes-templates
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+SHELL = /usr/bin/env bash
+
+.PHONY: default
+default: deploy
+ @:
+
+.PHONY: deploy
+deploy:
+ @./deploy.sh
diff --git a/gcp_cloud_function_ifconfig/README.md b/gcp_cloud_function_ifconfig/README.md
new file mode 100644
index 000000000..0485fcf08
--- /dev/null
+++ b/gcp_cloud_function_ifconfig/README.md
@@ -0,0 +1,45 @@
+Google Cloud Function - ifconfig
+=====================
+
+Queries http://ifconfig.co from GCF to check the routing and external IP being used eg. for comparison with Cloudflare / Firewall rules
+
+- `main.py` - the code
+- `requirements.txt` - the pip modules to bootstrap
+- `deploy.sh` - upload the code and deps
+
+Response is HTTP status code and message, then the raw JSON results
+
+```
+200 OK
+
+{
+ "ip": "1.2.3.4",
+ "ip_decimal": 1234567890,
+ "country": "United States",
+ "country_iso": "US",
+ "country_eu": false,
+ "latitude": 37.751,
+ "longitude": -97.822,
+ "time_zone": "America/Chicago",
+ "asn": "AS15169",
+ "asn_org": "GOOGLE",
+ "hostname": "ipv6.gae.googleusercontent.com",
+ "user_agent": {
+ "product": "python-requests",
+ "version": "2.24.0",
+ "raw_value": "python-requests/2.24.0"
+ }
+}
+```
+
+For IPv6 the format will be more like:
+```
+ "ip": "1234:5678:9012:34::a",
+ "ip_decimal": 12345678901234567890123456789012345678,
+```
+
+Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches:
+
+```
+./deploy.sh
+```
diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh
new file mode 100755
index 000000000..39aeba85e
--- /dev/null
+++ b/gcp_cloud_function_ifconfig/deploy.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+cd "$srcdir"
+
+name="ifconfig"
+
+# Cloud Functions not available in all regions yet:
+#
+# https://cloud.google.com/functions/docs/locations
+#
+# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment
+region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)"
+region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}"
+
+# https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com
+# for serverless VPC access to resources using their Private IPs
+# since we're only accessing the SQL Admin API we don't need this
+#vpc_connector="my-vpc-connector"
+
+opts=()
+if [ -n "${vpc_connector:-}" ]; then
+ # routes all traffic through VPC connector to re-use the VPC's Cloud NAT IP eg. for permitting in firewall rules
+ opts+=(--vpc-connector "$vpc_connector" --egress-settings all)
+fi
+
+set -x
+gcloud functions deploy "$name" --trigger-http \
+ --security-level=secure-always \
+ --runtime python39 \
+ --entry-point main \
+ --memory 128MB \
+ --region "$region" \
+ --timeout 60 \
+ "${opts[@]}" \
+ --quiet # don't prompt to --allow-unauthenticated
+ #--max-instances 1 \
diff --git a/gcp_cloud_function_ifconfig/main.py b/gcp_cloud_function_ifconfig/main.py
new file mode 100755
index 000000000..c9e50b0e7
--- /dev/null
+++ b/gcp_cloud_function_ifconfig/main.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+GCP Cloud Function to query ifconfig.co to show our IP information for debugging VPC Connector access
+routing via specified VPC Network to using default NAT Gateway
+
+Example usage: check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules
+
+The HTTP request is irrelevant, just pass an empty JSON document '{}', although we could pass the website to query
+in a field in which case this would just act as a proxy.
+
+See Also: gcp_cloud_function_proxy/main.py
+
+
+Tested on GCP Cloud Functions with Python 3.9
+
+"""
+
+# https://cloud.google.com/functions/docs/writing/http#writing_http_content-python
+
+# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python
+
+import requests
+
+def main(_):
+ """Responds to any HTTP request.
+ Args:
+ request (flask.Request): HTTP request object.
+ Returns:
+ The response text or any set of values that can be turned into a
+ Response object using
+ `make_response `.
+ """
+ req = requests.get('http://ifconfig.co/json') # show our IP information for debugging VPC connector routing
+ status_code = req.status_code
+ status_message = req.reason
+ content = req.text
+ return "{} {}\n\n{}".format(status_code, status_message, content)
diff --git a/gcp_cloud_function_ifconfig/requirements.txt b/gcp_cloud_function_ifconfig/requirements.txt
new file mode 100644
index 000000000..775d29ac6
--- /dev/null
+++ b/gcp_cloud_function_ifconfig/requirements.txt
@@ -0,0 +1,2 @@
+# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python
+requests==2.24.0
diff --git a/gcp_cloud_function_proxy/.gcloudignore b/gcp_cloud_function_proxy/.gcloudignore
new file mode 100644
index 000000000..42de3fd00
--- /dev/null
+++ b/gcp_cloud_function_proxy/.gcloudignore
@@ -0,0 +1,22 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020)
+#
+# vim:ts=4:sts=4:sw=4:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore
+
+.git
+.gcloudignore
+deploy.sh
+test/
+tests/
diff --git a/gcp_cloud_function_proxy/Makefile b/gcp_cloud_function_proxy/Makefile
new file mode 100644
index 000000000..656b78879
--- /dev/null
+++ b/gcp_cloud_function_proxy/Makefile
@@ -0,0 +1,22 @@
+#
+# Author: Hari Sekhon
+# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021)
+#
+# vim:ts=4:sts=4:sw=4:noet
+#
+# https://github.com/HariSekhon/Kubernetes-templates
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+SHELL = /usr/bin/env bash
+
+.PHONY: default
+default: deploy
+ @:
+
+.PHONY: deploy
+deploy:
+ @./deploy.sh
diff --git a/gcp_cloud_function_proxy/README.md b/gcp_cloud_function_proxy/README.md
new file mode 100644
index 000000000..1a4c2465a
--- /dev/null
+++ b/gcp_cloud_function_proxy/README.md
@@ -0,0 +1,27 @@
+Google Cloud Function - Proxy
+=====================
+
+Queries a given URL from GCF to check connectivity eg. for testing with Cloudflare / Firewall rules
+
+Query content:
+```
+{"url": "http://ifconfig.co/json"}
+```
+
+Response is HTTP status code and message, blank line and then the content:
+
+```
+200 OK
+
+
+```
+
+- `main.py` - the code
+- `requirements.txt` - the pip modules to bootstrap
+- `deploy.sh` - upload the code and deps
+
+Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches:
+
+```
+./deploy.sh
+```
diff --git a/gcp_cloud_function_proxy/deploy.sh b/gcp_cloud_function_proxy/deploy.sh
new file mode 100755
index 000000000..d8e04e9b5
--- /dev/null
+++ b/gcp_cloud_function_proxy/deploy.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+cd "$srcdir"
+
+name="proxy"
+
+# Cloud Functions not available in all regions yet:
+#
+# https://cloud.google.com/functions/docs/locations
+#
+# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment
+region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)"
+region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}"
+
+# https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com
+# for serverless VPC access to resources using their Private IPs
+# since we're only accessing the SQL Admin API we don't need this
+#vpc_connector="my-vpc-connector"
+
+opts=()
+if [ -n "${vpc_connector:-}" ]; then
+ # routes all traffic through VPC connector to re-use the VPC's Cloud NAT IP eg. for permitting in firewall rules
+ opts+=(--vpc-connector "$vpc_connector" --egress-settings all)
+fi
+
+set -x
+gcloud functions deploy "$name" --trigger-http \
+ --security-level=secure-always \
+ --runtime python39 \
+ --entry-point main \
+ --memory 128MB \
+ --region "$region" \
+ --timeout 60 \
+ "${opts[@]}" \
+ --quiet # don't prompt to --allow-unauthenticated
+ #--max-instances 1
diff --git a/gcp_cloud_function_proxy/main.py b/gcp_cloud_function_proxy/main.py
new file mode 100755
index 000000000..8680d455a
--- /dev/null
+++ b/gcp_cloud_function_proxy/main.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+GCP Cloud Function to query ifconfig.co to show our IP information for debugging VPC Connector access
+routing via specified VPC Network to using default NAT Gateway
+
+Example usage:
+
+Check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules
+
+Test request examples:
+
+ { "url": "http://ifconfig.co/json" }
+
+defaults to http:// if not specified:
+
+ { "url": "ifconfig.co/json" }
+
+
+Tested on GCP Cloud Functions with Python 3.9
+
+"""
+
+# https://cloud.google.com/functions/docs/writing/http#writing_http_content-python
+
+# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python
+
+import json
+import requests
+
+def main(request):
+ """Responds to any HTTP request.
+ Args:
+ request (flask.Request): HTTP request object.
+ Returns:
+ The response text or any set of values that can be turned into a
+ Response object using
+ `make_response `.
+ """
+ data = json.loads(request.data)
+ url = data['url']
+ if '://' not in url:
+ url = 'http://' + url
+ req = requests.get(url)
+ status_code = req.status_code
+ status_message = req.reason
+ content = req.text
+ return "{} {}\n\n{}".format(status_code, status_message, content)
diff --git a/gcp_cloud_function_proxy/requirements.txt b/gcp_cloud_function_proxy/requirements.txt
new file mode 100644
index 000000000..775d29ac6
--- /dev/null
+++ b/gcp_cloud_function_proxy/requirements.txt
@@ -0,0 +1,2 @@
+# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python
+requests==2.24.0
diff --git a/gcp_cloud_function_sql_export/.gcloudignore b/gcp_cloud_function_sql_export/.gcloudignore
new file mode 100644
index 000000000..42de3fd00
--- /dev/null
+++ b/gcp_cloud_function_sql_export/.gcloudignore
@@ -0,0 +1,22 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020)
+#
+# vim:ts=4:sts=4:sw=4:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore
+
+.git
+.gcloudignore
+deploy.sh
+test/
+tests/
diff --git a/gcp_cloud_function_sql_export/Makefile b/gcp_cloud_function_sql_export/Makefile
new file mode 100644
index 000000000..656b78879
--- /dev/null
+++ b/gcp_cloud_function_sql_export/Makefile
@@ -0,0 +1,22 @@
+#
+# Author: Hari Sekhon
+# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021)
+#
+# vim:ts=4:sts=4:sw=4:noet
+#
+# https://github.com/HariSekhon/Kubernetes-templates
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+SHELL = /usr/bin/env bash
+
+.PHONY: default
+default: deploy
+ @:
+
+.PHONY: deploy
+deploy:
+ @./deploy.sh
diff --git a/gcp_cloud_function_sql_export/README.md b/gcp_cloud_function_sql_export/README.md
new file mode 100644
index 000000000..db3ff61ad
--- /dev/null
+++ b/gcp_cloud_function_sql_export/README.md
@@ -0,0 +1,52 @@
+Google Cloud Function - SQL Backup Exporter to GCS
+=====================
+
+Triggers GCP [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage).
+
+Solution documentation:
+
+https://cloud.google.com/solutions/scheduling-cloud-sql-database-exports-using-cloud-scheduler
+
+- `main.py` - the code
+- `requirements.txt` - the pip modules to bootstrap
+- `deploy.sh` - upload the code and deps
+
+Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches:
+
+```
+./deploy.sh
+```
+
+### Solution Dependencies
+
+- a [Cloud PubSub](https://cloud.google.com/pubsub) topic
+- [Cloud Scheduler](https://cloud.google.com/scheduler) jobs to trigger backups
+ - see `gcp_cloud_schedule_sql_exports.sh` in [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo
+- a service account with permissions to access [Cloud SQL](https://cloud.google.com/sql)
+ - see `gcp_sql_create_readonly_service_account.sh` in [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo
+- each [Cloud SQL](https://cloud.google.com/sql) instance to be backed up requires objectCreator permissions to the [GCS](https://cloud.google.com/storage) bucket
+ - see `gcp_sql_grant_instances_gcs_object_creator.sh` in [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo
+
+### Serverless Framework
+
+Instead of `deploy.sh` you can alternatively use the [Serverless](https://www.serverless.com/) framework for which a `serverless.yml` config is provided:
+
+```
+serverless deploy
+```
+
+If this is your first time using Serverless then you'll need to install the GCP plugin:
+
+```
+serverless plugin install --name serverless-google-cloudfunctions
+```
+
+The `serverless.yml` config expects to find `$GOOGLE_PROJECT_ID` and `$GOOGLE_REGION` environment variables.
+
+Serverless requires additional permissions for the service account: Deployment Manager Editor and Storage Admin to create deployments and staging buckets.
+
+You can also build a serverless artifact to `.serverless/` without deploying it (generates Google [Deployment Manager](https://cloud.google.com/deployment-manager) templates and a zip file - useful to check what would be uploaded / ignored):
+
+```
+serverless package
+```
diff --git a/gcp_cloud_function_sql_export/deploy.sh b/gcp_cloud_function_sql_export/deploy.sh
new file mode 100755
index 000000000..c5c3c3249
--- /dev/null
+++ b/gcp_cloud_function_sql_export/deploy.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-10-16 10:12:26 +0100 (Fri, 16 Oct 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+cd "$srcdir"
+
+# needed to define the $service_account further down
+project="${CLOUDSDK_CORE_PROJECT:-$(gcloud config list --format="value(core.project)")}"
+
+# Cloud Functions not available in all regions yet:
+#
+# https://cloud.google.com/functions/docs/locations
+#
+# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment
+region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)"
+region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}"
+
+name="cloud-sql-backups"
+topic="cloud-sql-backups"
+service_account="cloud-function-sql-backup@$project.iam.gserviceaccount.com"
+# https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com
+# for serverless VPC access to resources using their Private IPs
+# since we're only accessing the SQL Admin API we don't need this
+#vpc_connector="cloud-sql-backups"
+
+gcloud functions deploy "$name" --trigger-topic "$topic" \
+ --runtime python37 \
+ --entry-point main \
+ --service-account "$service_account" \
+ --region "$region" \
+ --memory 128MB \
+ --timeout 60
+ # may want multiple concurrent calls to different SQL instances at the same time
+ #
+ # also doesn't prevent:
+ #
+ # "Operation failed because another operation was already in progress."
+ #
+ #--max-instances 1 # this isn't good enough because it sets off an async API call, so successive calls can fail if called before that SQL Admin API export has finished
+
+ # --vpc-connector "$vpc_connector"
diff --git a/gcp_cloud_function_sql_export/main.py b/gcp_cloud_function_sql_export/main.py
new file mode 100755
index 000000000..6e3765667
--- /dev/null
+++ b/gcp_cloud_function_sql_export/main.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-10-14 15:29:38 +0100 (Wed, 14 Oct 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+GCP Cloud Function to export a given Cloud SQL database to GCS via PubSub notifications from Cloud Scheduler
+
+Solution Documentation:
+
+ https://cloud.google.com/solutions/scheduling-cloud-sql-database-exports-using-cloud-scheduler
+
+GCP Cloud PubSub should be sent payloads like this by Cloud Scheduler (replacing the env vars with your literals):
+
+{
+ "project": "${GOOGLE_PROJECT_ID}",
+ "instance": "${SQL_INSTANCE_HOST}",
+ "database": "${DATABASE}",
+ "bucket": "${BUCKET_NAME}"
+}
+
+Tested on GCP Cloud Functions with Python 3.7
+
+"""
+
+# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python
+
+# Code below is based on solution sample code from the link above
+
+#import os
+import base64
+import logging
+import json
+
+from datetime import datetime
+from httplib2 import Http
+
+from googleapiclient import discovery
+from googleapiclient.errors import HttpError
+from oauth2client.client import GoogleCredentials
+
+
+# pylint: disable=unused-argument
+def main(event, context):
+# if os.getenv("DEBUG"):
+# # debug level logs don't appear in function details logs tab even with
+# # DEBUG=1 runtime env var set and logs severity filter set to >= DEBUG
+# #logging.debug('event: %s', event)
+# #logging.debug('context: %s', context)
+# logging.info('event: %s', event)
+# logging.info('context: %s', context)
+ data = json.loads(base64.b64decode(event['data']).decode('utf-8'))
+ credentials = GoogleCredentials.get_application_default()
+
+ service = discovery.build('sqladmin', 'v1beta4', http=credentials.authorize(Http()), cache_discovery=False)
+
+ project = data['project']
+ bucket = data['bucket']
+ bucket = bucket.lstrip('gs://')
+ instance = data['instance']
+ database = data['database']
+ timestamp = datetime.now().strftime("%Y-%m-%d_%H%M")
+
+ # .gz extension so it is auto-compressed, saving storage space + billing
+ backup_uri = "gs://{bucket}/backups/sql/{instance}--{database}--{timestamp}.sql.gz".format(
+ bucket=bucket,
+ instance=instance,
+ database=database,
+ timestamp=timestamp)
+
+ instances_export_request_body = {
+ "exportContext": {
+ "kind": "sql#exportContext",
+ "fileType": "SQL",
+ "uri": backup_uri,
+ "databases": [
+ database
+ ]
+ }
+ }
+
+ try:
+ logging.info("Requesting project '%s' database instance '%s' runs a backup export to bucket '%s' path '%s'",
+ project,
+ instance,
+ bucket,
+ backup_uri)
+ request = service.instances().export(
+ project=project,
+ instance=instance,
+ body=instances_export_request_body
+ )
+ response = request.execute()
+ except HttpError as err:
+ logging.error("Backup FAILED. Reason: %s", err)
+ else:
+ logging.info("Backup Task triggered: %s", response)
diff --git a/gcp_cloud_function_sql_export/requirements.txt b/gcp_cloud_function_sql_export/requirements.txt
new file mode 100644
index 000000000..2f7349f4d
--- /dev/null
+++ b/gcp_cloud_function_sql_export/requirements.txt
@@ -0,0 +1,3 @@
+# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python
+google-api-python-client==1.12.5
+oauth2client==4.1.3
diff --git a/gcp_cloud_function_sql_export/serverless.yml b/gcp_cloud_function_sql_export/serverless.yml
new file mode 100644
index 000000000..a73986efb
--- /dev/null
+++ b/gcp_cloud_function_sql_export/serverless.yml
@@ -0,0 +1,137 @@
+# vim:ts=2:sts=2:sw=2:et
+#
+# Author: Hari Sekhon
+# Date: 2020-10-21 11:19:06 +0100 (Wed, 21 Oct 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# S e r v e r l e s s
+# ============================================================================ #
+
+# Deploys the Google Cloud Function with a name --
+#
+# eg. cloud-sql-backups-production-main
+
+# Requires the same permissions as the simpler adjacent deploy.sh script but also:
+#
+# Deployment Manager Editor
+# Storage Admin (Storage Object Admin is not enough as it needs to create staging buckets and you'll get weird errors otherwise)
+#
+# for the credential file's service account which is also used to run the Cloud Function
+
+# Check generated config after environment variable interpolation:
+#
+# cd "$(dirname $0)" && serverless print
+#
+# My advanced vimrc has a hotkey for this:
+#
+# https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.vimrc
+
+frameworkVersion: '2'
+plugins:
+ - serverless-google-cloudfunctions
+
+# to prevent:
+#
+# Serverless: Configuration warning: Unrecognized provider 'google'
+# Serverless:
+# Serverless: You're relying on provider plugin which doesn't provide a validation schema for its config.
+configValidationMode: off
+# after the GCP plugin matures, switch to
+#configValidationMode: error
+
+service: cloud-sql-backups
+provider:
+ name: google
+ stage: production
+ runtime: python37
+ region: ${env:GOOGLE_REGION, "europe-west1"}
+ project: ${env:GOOGLE_PROJECT_ID}
+
+ # https://serverless.com/framework/docs/providers/google/guide/credentials/
+ #
+ # path to credentials file needs to be absolute
+ #
+ # download a credentials file for the service account:
+ #
+ # https://cloud.google.com/iam/docs/creating-managing-service-account-keys#iam-service-account-keys-create-gcloud
+ #
+ # mkdir -pv ~/.gcloud && gcloud iam service-accounts keys create ~/.gcloud/cloud-function-sql-backup-keyfile.json --iam-account "cloud-function-sql-backup@$(gcloud config list --format="get(core.project)").iam.gserviceaccount.com"
+ #
+ credentials: ~/.gcloud/cloud-function-sql-backup-keyfile.json
+ #
+ # or use personal creds - see $GOOGLE_APPLICATION_CREDENTIALS or generate:
+ #
+ # gcloud auth application-default login # [ --client-id-file=~/.gcloud/keyfile.json ]
+ #
+ #credentials: ~/.config/gcloud/application_default_credentials.json
+ #credentials: ${env:HOME}/.config/gcloud/application_default_credentials.json
+ #credentials: /Users/harisekhon/.config/gcloud/application_default_credentials.json
+
+# https://www.serverless.com/framework/docs/providers/google/guide/packaging/
+package:
+ exclude:
+ ## needs more granular excluding in production as only the serverless provider npm
+ ## package should be excluded (and not the whole node_modules directory)
+ #- node_modules/**
+ #- deploy.sh
+ #- test/**
+ #- tests/**
+ #- .gitignore
+ #- .gcloudignore
+ #- .git/**
+ #- package.json
+ #- package-lock.json
+ #- README.md
+ # XXX: exclude all hidden files and directories
+ - .*
+ # XXX: more robust to only whitelist include as per .dockerignore best practice too (unfortunately not supported in .gcloudignore)
+ - ./**
+ include:
+ - main.py
+ - requirements.txt
+ # or specify your own zipfile and skip packaging:
+ #artifact: path/to/my-artifact.zip
+
+# https://www.serverless.com/framework/docs/providers/google/guide/functions/
+#
+# use an array of includes for bigger serverless deployments of multiple functions
+#functions:
+# - ${file(../foo-functions.yml)}
+# - ${file(../bar-functions.yml)}
+#
+functions:
+ #myfunc:
+ # handler: http
+ # events:
+ # - http: path
+ # NOTE: the following uses an "event" event (pubSub event in this case).
+ # Please create the corresponding resources in the Google Cloud
+ # before deploying this service through Serverless
+ main:
+ handler: main
+ memorySize: 128
+ timeout: 60s
+ events:
+ - event:
+ eventType: providers/cloud.pubsub/eventTypes/topic.publish
+ #resource: projects/*/topics/my-topic
+ resource: projects/${env:GOOGLE_PROJECT_ID}/topics/cloud-sql-backups
+# you can define resources, templates etc. the same way you would in a
+# Google Cloud deployment configuration
+resources:
+ resources:
+ # generates the bucket that the cloud function will send the backups to
+ # there is another bucket also created for uploading the package (sls-cloud-sql-backups-production-1603283101338), not to be confused with this one
+ - type: storage.v1.bucket
+ name: ${env:GOOGLE_PROJECT_ID}-sql-backups
+# imports:
+# - path: my_template.jinja
diff --git a/gcp_service_account_credential_keys.py b/gcp_service_account_credential_keys.py
new file mode 100755
index 000000000..f58f86e98
--- /dev/null
+++ b/gcp_service_account_credential_keys.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-10-29 18:02:14 +0000 (Thu, 29 Oct 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Lists all service account credential keys in a given GCP project
+
+Excludes built-in system managed keys which are hidden in the Console UI anyway and are not actionable
+or in scope for a key policy audit.
+
+
+Output Format:
+
+
+
+
+You can supply a service account credentials file to authenticate with or just use ADC via:
+
+ gcloud auth application-default login
+
+
+See Also - similar scripts in the DevOps Bash tools repo:
+
+ https://github.com/HariSekhon/DevOps-Bash-tools/
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from datetime import datetime
+import json
+import os
+import sys
+import traceback
+from google.oauth2 import service_account
+import googleapiclient.discovery
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, log_option, validate_int
+ from harisekhon import CLI
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1'
+
+
+class GcpServiceAccountCredentialKeys(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(GcpServiceAccountCredentialKeys, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.credentials_file = None
+ self.service = None
+ self.project = None
+ self.no_expiry = None
+ self.expired = None
+ self.expires_within_days = None
+
+ def add_options(self):
+ super(GcpServiceAccountCredentialKeys, self).add_options()
+ self.add_opt('-f', '--credentials-file', metavar='',
+ default=os.getenv('GOOGLE_CREDENTIALS', \
+ os.getenv('GOOGLE_APPLICATION_CREDENTIALS')),
+ help='Credentials file ($GOOGLE_CREDENTIALS, ' + \
+ '$GOOGLE_APPLICATION_CREDENTIALS)')
+ self.add_opt('-p', '--project-id', metavar='',
+ help='Google Cloud Project ID ($GOOGLE_PROJECT_ID, or inferred from credentials file')
+ self.add_opt('-n', '--no-expiry', action='store_true', help='List only non-expiring keys')
+ self.add_opt('-e', '--expired', action='store_true', help='List only expired keys')
+ self.add_opt('-d', '--expires-within-days', type=int, help='List only keys that will expire within N days')
+
+ def process_options(self):
+ super(GcpServiceAccountCredentialKeys, self).process_options()
+ self.no_args()
+ project_id = self.get_opt('project_id')
+ credsfile = self.get_opt('credentials_file')
+ self.no_expiry = self.get_opt('no_expiry')
+ self.expired = self.get_opt('expired')
+ self.expires_within_days = self.get_opt('expires_within_days')
+ #if not credsfile:
+ # self.usage('no --credentials-file given and ' + \
+ # 'GOOGLE_CREDENTIALS / GOOGLE_APPLICATION_CREDENTIALS environment variables not populated')
+ if credsfile:
+ if not os.path.exists(credsfile):
+ self.usage('credentials file not found: {}'.format(credsfile))
+ self.credentials_file = credsfile
+ log_option('credentials file', self.credentials_file)
+ if not project_id:
+ if credsfile:
+ json_data = json.loads(open(credsfile).read())
+ project_id = json_data['project_id']
+ else:
+ self.usage('--project-id not specified and no credentials file given from which to infer')
+ self.project = project_id
+ log_option('project', self.project)
+ if self.expires_within_days is not None:
+ validate_int(self.expires_within_days, 'expires within days', 0)
+ if self.no_expiry:
+ self.usage('--expires-within-days and --no-expiry are mutually exclusive')
+ if self.expired:
+ self.usage('--expires-within-days and --expired are mutually exclusive')
+ if self.no_expiry and self.expired:
+ self.usage('--expired and --no-expiry are mutually exclusive')
+
+ def run(self):
+ # defaults to looking for $GOOGLE_APPLICATION_CREDENTIALS or using Application Default Credentials
+ # from 'gcloud auth application-default login' => ~/.config/gcloud/application_default_credentials.json
+ credentials = None
+ if self.credentials_file:
+ log.debug('loading credentials')
+ credentials = service_account.Credentials.from_service_account_file(
+ filename=self.credentials_file,
+ scopes=['https://www.googleapis.com/auth/cloud-platform']
+ )
+
+ # cache_discovery=False avoids:
+ # ImportError: file_cache is unavailable when using oauth2client >= 4.0.0 or google-auth
+ self.service = googleapiclient.discovery.build('iam', 'v1', credentials=credentials, cache_discovery=False)
+
+ for service_account_email in self.get_service_accounts():
+ self.list_keys(service_account_email)
+
+ def get_service_accounts(self):
+ """ Returns a list of service account email addresses """
+
+ log.debug('getting service accounts')
+ service_accounts = self.service.projects()\
+ .serviceAccounts()\
+ .list(name='projects/{}'.format(self.project))\
+ .execute()
+ for account in service_accounts['accounts']:
+ yield account['email']
+
+ def list_keys(self, service_account_email):
+ log.debug("getting keys for service account '%s'", service_account_email)
+ keys = self.service.projects()\
+ .serviceAccounts()\
+ .keys()\
+ .list(name='projects/-/serviceAccounts/' + service_account_email).execute()
+
+ for key in keys['keys']:
+ if key['keyType'] == 'SYSTEM_MANAGED':
+ continue
+ _id = key['name'].split('/')[-1]
+ created_date = key['validAfterTime']
+ expiry_date = key['validBeforeTime']
+ created_datetime = datetime.strptime(created_date, "%Y-%m-%dT%H:%M:%SZ")
+ expiry_datetime = datetime.strptime(expiry_date, "%Y-%m-%dT%H:%M:%SZ")
+ age_timedelta = datetime.utcnow() - created_datetime
+ age_days = int(age_timedelta.total_seconds() / 86400)
+ expired = False
+ if expiry_date == '9999-12-31T23:59:59Z':
+ expires_in_days = 'NEVER'
+ else:
+ expires_in_timedelta = expiry_datetime - datetime.utcnow()
+ expires_in_days = int(expires_in_timedelta.total_seconds() / 86400)
+ if expires_in_days < 1:
+ expired = True
+ if self.no_expiry and (expires_in_days != 'NEVER'):
+ continue
+ if self.expired and expired:
+ continue
+ if self.expires_within_days is not None and \
+ (expires_in_days == 'NEVER' or expires_in_days > self.expires_within_days):
+ continue
+ print('{id} {created} {expires} {age:4d} {expires_in:5s} {expired} {service_account}'.format(
+ id=_id,
+ created=created_date,
+ expires=expiry_date,
+ age=age_days,
+ expires_in=expires_in_days,
+ expired=expired,
+ service_account=service_account_email
+ ))
+
+
+if __name__ == '__main__':
+ GcpServiceAccountCredentialKeys().main()
diff --git a/getent.py b/getent.py
index ce77aefcf..d8b701cd6 100755
--- a/getent.py
+++ b/getent.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-11-20 12:35:49 +0000 (Sun, 20 Nov 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/git_check_branches_upstream.py b/git_check_branches_upstream.py
index 0ab2e177b..27ec09865 100755
--- a/git_check_branches_upstream.py
+++ b/git_check_branches_upstream.py
@@ -1,24 +1,24 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-07-21 16:19:19 +0100 (Thu, 21 Jul 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
Tool to check Git branches have their upstream origin branch set consistently and auto-fix if necessary
-Mainly written for my https://github.com/harisekhon/Dockerfiles repo
+Mainly written for my https://github.com/HariSekhon/Dockerfiles repo
which has over 100 branches which get merged, pulled and pushed around
"""
diff --git a/hbase_compact_tables.py b/hbase_compact_tables.py
index 60f246328..7bfca40b2 100755
--- a/hbase_compact_tables.py
+++ b/hbase_compact_tables.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-04-27 20:49:23 +0100 (Wed, 27 Apr 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_flush_tables.py b/hbase_flush_tables.py
index c4f43ef8d..71be25c17 100755
--- a/hbase_flush_tables.py
+++ b/hbase_flush_tables.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-09-12 13:44:24 +0200 (Mon, 12 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# NOTE: 'flush' is not supported in Happybase as it's not in the Thrift API,
@@ -44,7 +44,7 @@
import sys
import traceback
import subprocess
-from subprocess import PIPE
+PIPE = subprocess.PIPE
libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
sys.path.append(libdir)
try:
diff --git a/hbase_generate_data.py b/hbase_generate_data.py
index 133fa42dc..3ed31a644 100755
--- a/hbase_generate_data.py
+++ b/hbase_generate_data.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-09-14 15:19:35 +0200 (Wed, 14 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_region_requests.py b/hbase_region_requests.py
index 2af24c219..d182431c8 100755
--- a/hbase_region_requests.py
+++ b/hbase_region_requests.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_regions_by_memstore_size.py b/hbase_regions_by_memstore_size.py
index 47165a7a9..295653a81 100755
--- a/hbase_regions_by_memstore_size.py
+++ b/hbase_regions_by_memstore_size.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_regions_by_size.py b/hbase_regions_by_size.py
index 49f7419d8..c67184c8b 100755
--- a/hbase_regions_by_size.py
+++ b/hbase_regions_by_size.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_regions_least_used.py b/hbase_regions_least_used.py
index 45727dd34..752be4ed5 100755
--- a/hbase_regions_least_used.py
+++ b/hbase_regions_least_used.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_regionserver_requests.py b/hbase_regionserver_requests.py
index f61b40cbd..a8c96ebb9 100755
--- a/hbase_regionserver_requests.py
+++ b/hbase_regionserver_requests.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_scan_table_column_names.sh b/hbase_scan_table_column_names.sh
index f2d62ad44..026f7d5a8 100755
--- a/hbase_scan_table_column_names.sh
+++ b/hbase_scan_table_column_names.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-06-29 19:01:22 +0100 (Fri, 29 Jun 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/hbase_show_table_region_ranges.py b/hbase_show_table_region_ranges.py
index 2a5647fb9..585e11cf9 100755
--- a/hbase_show_table_region_ranges.py
+++ b/hbase_show_table_region_ranges.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-05 13:57:37 +0100 (Wed, 05 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_table_region_row_distribution.py b/hbase_table_region_row_distribution.py
index 87b4d2e34..0dfe6e024 100755
--- a/hbase_table_region_row_distribution.py
+++ b/hbase_table_region_row_distribution.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_table_regions_by_regionserver.sh b/hbase_table_regions_by_regionserver.sh
index 5e603e289..8017f406a 100755
--- a/hbase_table_regions_by_regionserver.sh
+++ b/hbase_table_regions_by_regionserver.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-08-24 19:34:33 +0100 (Fri, 24 Aug 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/hbase_table_regionserver_distribution.sh b/hbase_table_regionserver_distribution.sh
index 252a9a3d4..58699a877 100755
--- a/hbase_table_regionserver_distribution.sh
+++ b/hbase_table_regionserver_distribution.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-11 19:36:23 +0100 (Wed, 11 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# Useful to see table region skew across regionservers
diff --git a/hbase_table_row_key_distribution.py b/hbase_table_row_key_distribution.py
index 8784d5952..6059763d7 100755
--- a/hbase_table_row_key_distribution.py
+++ b/hbase_table_row_key_distribution.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-08 09:02:01 +0100 (Sat, 08 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_uncompacted_regions.sh b/hbase_uncompacted_regions.sh
index 7b092c06d..00a3302c5 100755
--- a/hbase_uncompacted_regions.sh
+++ b/hbase_uncompacted_regions.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-11 19:26:21 +0100 (Wed, 11 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# Find regions that require major_compact after region migration to achieve data locality again
diff --git a/hadoop_hdfs_files_native_checksums.jy b/hdfs_files_native_checksums.jy
similarity index 98%
rename from hadoop_hdfs_files_native_checksums.jy
rename to hdfs_files_native_checksums.jy
index 6aa6e6ff2..9c558401a 100755
--- a/hadoop_hdfs_files_native_checksums.jy
+++ b/hdfs_files_native_checksums.jy
@@ -3,7 +3,7 @@
# Author: Hari Sekhon
# Date: 2013-06-20 18:21:02 +0100 (Thu, 20 Jun 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
@@ -50,7 +50,7 @@ def usage(*msg):
if msg:
printerr("".join(msg))
die("""
-Hari Sekhon - https://github.com/harisekhon/devops-python-tools
+Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools
================================================================================
%s - version %s
diff --git a/hadoop_hdfs_files_stats.jy b/hdfs_files_stats.jy
similarity index 98%
rename from hadoop_hdfs_files_stats.jy
rename to hdfs_files_stats.jy
index 69132daee..587b3b390 100755
--- a/hadoop_hdfs_files_stats.jy
+++ b/hdfs_files_stats.jy
@@ -3,7 +3,7 @@
# Author: Hari Sekhon
# Date: 2013-06-08 22:06:27 +0100 (Sat, 08 Jun 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
@@ -55,7 +55,7 @@ def usage(*msg):
if msg:
printerr(msg)
die("""
-Hari Sekhon - https://github.com/harisekhon/devops-python-tools
+Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools
================================================================================
%s - version %s
diff --git a/hdfs_find_replication_factor_1.py b/hdfs_find_replication_factor_1.py
index fa80c3586..1f348556d 100755
--- a/hdfs_find_replication_factor_1.py
+++ b/hdfs_find_replication_factor_1.py
@@ -1,18 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# coding=utf-8
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2018-11-28 16:37:00 +0000 (Wed, 28 Nov 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -123,8 +123,7 @@ def run(self):
print('', file=sys.stderr)
print(file_path)
if self.replication_factor:
- log.info('setting replication factor to {} on {}'\
- .format(self.replication_factor, file_path))
+ log.info('setting replication factor to %s on %s', self.replication_factor, file_path)
# returns a generator so must evaluate in order to actually execute
# otherwise you find there is no effect on the replication factor
for _ in client.setrep([file_path], self.replication_factor, recurse=False):
diff --git a/hadoop_hdfs_time_block_reads.jy b/hdfs_time_block_reads.jy
similarity index 99%
rename from hadoop_hdfs_time_block_reads.jy
rename to hdfs_time_block_reads.jy
index e340cff7c..ab1812a23 100755
--- a/hadoop_hdfs_time_block_reads.jy
+++ b/hdfs_time_block_reads.jy
@@ -3,7 +3,7 @@
# Author: Hari Sekhon
# Date: 2013-06-08 22:06:27 +0100 (Sat, 08 Jun 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
@@ -28,7 +28,7 @@ __version__ = '0.9.1'
import os
usage_msg = """
-Hari Sekhon - https://github.com/harisekhon/devops-python-tools
+Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools
================================================================================
%s - version %s
diff --git a/headtail.py b/headtail.py
index ff061445a..ec21a3f5c 100755
--- a/headtail.py
+++ b/headtail.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-01-07 22:57:18 +0000 (Thu, 07 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish # pylint: disable=line-too-long
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -41,7 +41,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.3.1'
+__version__ = '0.3.2'
class HeadTail(CLI):
@@ -60,6 +60,7 @@ def __init__(self):
self.sep = '-' * 80
self.docsep = '=' * 80
self.quiet = False
+ self.timeout_default = None
def add_options(self):
#self.timeout_default = 300
diff --git a/hexanonymize.py b/hexanonymize.py
new file mode 100755
index 000000000..cac35e6a0
--- /dev/null
+++ b/hexanonymize.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-01-02 17:08:32 +0000 (Thu, 02 Jan 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: GNU GPL version 2 (this file only), rest of this repo is licensed as per the adjacent LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tool to anonymize hex input but keeping the structure of the positions of numbers and digits
+
+Useful to retain the structure of ID formats
+
+Reads any given files or standard input and replaces each hex character with an incrementing number of letter (a-f)
+printing to stdout for piping or redirecting to a file as per unix filter command standards
+
+Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from
+standard input
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import os
+import sys
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+
+class HexAnonymize(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(HexAnonymize, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.preserve_case = False
+ self.only_hex_alphas = False
+
+ def add_options(self):
+ super(HexAnonymize, self).add_options()
+ self.add_opt('-c', '--case', action='store_true', help='Preserve case')
+ self.add_opt('-o', '--hex-only', action='store_true',
+ help='Only replace hex alpha chars (A-F, a-f), otherwise replaces all alphanumerics for safety')
+
+ def process_options(self):
+ super(HexAnonymize, self).process_options()
+ self.preserve_case = self.get_opt('case')
+ self.only_hex_alphas = self.get_opt('hex_only')
+
+ def hexanonymize(self, filehandle):
+ preserve_case = self.preserve_case
+ only_hex_alphas = self.only_hex_alphas
+ hex_alphas = ['a', 'b', 'c', 'd', 'e', 'f']
+ for line in filehandle:
+ integer = 1
+ letter = 'a'
+ for char in line:
+ if char.isdigit():
+ char = integer
+ integer += 1
+ if integer > 9:
+ integer = 0
+ elif (not only_hex_alphas and char.isalpha()) or char.lower() in hex_alphas:
+ if preserve_case and char.isupper():
+ char = letter.upper()
+ else:
+ char = letter
+ letter = chr(ord(char) + 1)
+ if letter.lower() not in hex_alphas:
+ letter = 'a'
+ print(char, end='')
+
+
+ def run(self):
+ if not self.args:
+ self.args.append('-')
+ for arg in self.args:
+ if arg == '-':
+ continue
+ if not os.path.exists(arg):
+ print("'%s' not found" % arg)
+ sys.exit(1)
+ for arg in self.args:
+ if arg == '-':
+ self.hexanonymize(sys.stdin)
+ else:
+ with open(arg) as filehandle:
+ self.hexanonymize(filehandle)
+
+
+if __name__ == '__main__':
+ HexAnonymize().main()
diff --git a/hive_compute_table_stats.py b/hive_compute_table_stats.py
new file mode 100755
index 000000000..d416f9379
--- /dev/null
+++ b/hive_compute_table_stats.py
@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to a HiveServer2 and compute optimization statistics on all tables,
+or only those matching given db / table / partition value regexes
+
+Tested on CDH 5.10, Hive 1.1.0 with Kerberos
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
+import logging
+import os
+import re
+import socket
+import sys
+# pylint: disable=import-error
+from impala.dbapi import connect
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+logging.basicConfig()
+log = logging.getLogger(os.path.basename(sys.argv[0]))
+
+host_envs = [
+ 'HIVESERVER2_HOST',
+ 'HIVE_HOST',
+ 'HOST'
+]
+
+port_envs = [
+ 'HIVESERVER2_PORT',
+ 'HIVE_PORT',
+ 'PORT'
+]
+
+def getenvs(keys, default=None):
+ for key in keys:
+ value = os.getenv(key)
+ if value:
+ return value
+ return default
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description="Computes statistics on all Hive tables/partitions matching database / table / partition regexes")
+ parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\
+ help='HiveServer2 host ' + \
+ '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')')
+ parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000),
+ help='HiveServer2 port (default: 10000, ' + ', $'.join(port_envs) + ')')
+ parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)')
+ parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)')
+ parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)')
+ parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)')
+ parser.add_argument('-n', '--krb5-service-name', default='hive',
+ help='Service principal (default: \'hive\')')
+ parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL')
+ parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode')
+ args = parser.parse_args()
+
+ if args.verbose:
+ log.setLevel(logging.INFO)
+
+ return args
+
+def connect_db(args, database):
+ auth_mechanism = None
+ if args.kerberos:
+ auth_mechanism = 'GSSAPI'
+
+ log.info('connecting to %s:%s database %s', args.host, args.port, database)
+ return connect(
+ host=args.host,
+ port=args.port,
+ auth_mechanism=auth_mechanism,
+ use_ssl=args.ssl,
+ #user=user,
+ #password=password,
+ database=database,
+ kerberos_service_name=args.krb5_service_name
+ )
+
+def main():
+ args = parse_args()
+
+ try:
+ database_regex = re.compile(args.database, re.I)
+ table_regex = re.compile(args.table, re.I)
+ partition_regex = re.compile(args.partition, re.I)
+ except re.error as _:
+ log.error('error in provided regex: %s', _)
+ sys.exit(3)
+
+ conn = connect_db(args, None)
+
+ log.info('querying databases')
+ with conn.cursor() as db_cursor:
+ db_cursor.execute('show databases')
+ for db_row in db_cursor:
+ database = db_row[0]
+ if not database_regex.search(database):
+ log.debug("skipping database '%s', does not match regex '%s'", database, args.database)
+ continue
+ log.info('querying tables for database %s', database)
+ #db_conn = connect_db(args, database)
+ #with db_conn.cursor() as table_cursor:
+ with conn.cursor() as table_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #table_cursor.execute('use %(database)s', {'database': database})
+ table_cursor.execute('use {}'.format(database))
+ table_cursor.execute('show tables')
+ for table_row in table_cursor:
+ table = table_row[0]
+ if not table_regex.search(table):
+ log.debug("skipping database '%s' table '%s', does not match regex '%s'", \
+ database, table, args.table)
+ continue
+ compute_table_stats(conn, args, database, table, partition_regex)
+
+def compute_table_stats(conn, args, database, table, partition_regex):
+ log.info("getting partitions for database '%s' table '%s'", database, table)
+ partitions_found = False
+ with conn.cursor() as partition_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ partition_cursor.execute('use {}'.format(database))
+ partition_cursor.execute('show partitions {}'.format(database))
+ for partitions_row in partition_cursor:
+ partition_key = partitions_row[0]
+ partition_value = partitions_row[1]
+ partitions_found = True
+ if not partition_regex.match(partition_value):
+ # pylint: disable=logging-not-lazy
+ log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " +
+ "value does not match regex '%s'",
+ database,
+ table,
+ partition_key,
+ partition_value,
+ args.partition)
+ continue
+ # doesn't support parameterized query quoting from dbapi spec
+ partition_cursor.execute('ANALYZE TABLE {db}.{table} PARTITION({key}={value}) COMPUTE STATISTICS'\
+ .format(db=database, table=table, key=partition_key, value=partition_value))
+ if not partitions_found:
+ log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table)
+ with conn.cursor() as table_cursor:
+ log.info("running compute stats on table '%s'", table)
+ # doesn't support parameterized query quoting from dbapi spec
+ table_cursor.execute('ANALYZE TABLE {db}.{table} COMPUTE STATISTICS'.format(db=database, table=table))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/hive_foreach_table.py b/hive_foreach_table.py
new file mode 100755
index 000000000..9d5abf6d6
--- /dev/null
+++ b/hive_foreach_table.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and execute a query for each table in each database,
+or only those matching given db / table regexes
+
+Useful for getting row counts of all tables or analyzing tables:
+
+eg.
+
+hive_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}'
+hive_foreach_table.py --query 'ANALYZE TABLE {db}.{table} COMPUTE STATS'
+
+or just for today's partition:
+
+hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(date=$(date '+%Y-%m-%d')) COMPUTE STATS"
+
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import re
+import sys
+import impala
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_regex
+ from hive_impala_cli import HiveImpalaCLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.2'
+
+
+class HiveForEachTable(HiveImpalaCLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveForEachTable, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # self.query can be pre-defined in which case subclassed programs won't expose --query option
+ self.query = None
+ self.database = None
+ self.table = None
+ self.partition = None
+ self.ignore_errors = False
+ self.table_count = 0
+
+ def add_options(self):
+ super(HiveForEachTable, self).add_options()
+ # allow subclassing to pre-define query and not expose option in that case
+ if self.query is None:
+ self.add_opt('-q', '--query', help='Query or statement to execute for each table' + \
+ ' (replaces {db} and {table} in the query string with each table and its database)')
+ self.add_opt('-d', '--database', default='.*', help='Database regex (default: .*)')
+ self.add_opt('-t', '--table', default='.*', help='Table regex (default: .*)')
+ #self.add_opt('-p', '--partition', default='.*', help='Partition regex (default: .*)')
+#
+# ignore tables that fail with errors like:
+#
+# Hive (CDH has MR, no tez):
+#
+# impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long
+#
+# Impala:
+#
+# impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table ''
+# CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table ''
+#
+ self.add_opt('-e', '--ignore-errors', action='store_true', help='Ignore individual table errors and continue')
+
+ def process_options(self):
+ super(HiveForEachTable, self).process_options()
+ if self.query is None:
+ self.query = self.get_opt('query')
+ if not self.query:
+ self.usage('query not defined')
+ self.database = self.get_opt('database')
+ self.table = self.get_opt('table')
+ #self.partition = self.get_opt('partition')
+ self.ignore_errors = self.get_opt('ignore_errors')
+ validate_regex(self.database, 'database')
+ validate_regex(self.table, 'table')
+ #validate_regex(self.partition, 'partition')
+
+ def run(self):
+ database_regex = re.compile(self.database, re.I)
+ table_regex = re.compile(self.table, re.I)
+ #partition_regex = re.compile(self.partition, re.I)
+ conn = self.connect('default')
+ log.info('querying databases')
+ # collecting in local list because long time iteration results in
+ # impala.error.HiveServer2Error: Invalid query handle
+ databases = []
+ database_count = 0
+ with conn.cursor() as db_cursor:
+ db_cursor.execute('show databases')
+ for db_row in db_cursor:
+ database = db_row[0]
+ database_count += 1
+ if not database_regex.search(database):
+ log.debug("skipping database '%s', does not match regex '%s'", database, self.database)
+ continue
+ databases.append(database)
+ log.info('%s/%s databases selected', len(databases), database_count)
+ for database in databases:
+ self.process_database(database, table_regex)
+ log.info('processed %s databases, %s tables', database_count, self.table_count)
+
+ def process_database(self, database, table_regex):
+ tables = []
+ table_count = 0
+ log.info("querying tables for database '%s'", database)
+ conn = self.connect(database)
+ with conn.cursor() as table_cursor:
+ try:
+ # doesn't support parameterized query quoting from dbapi spec
+ #table_cursor.execute('use %(database)s', {'database': database})
+ table_cursor.execute('use `{}`'.format(database))
+ table_cursor.execute('show tables')
+ except impala.error.HiveServer2Error as _:
+ log.error("error querying tables for database '%s': %s", database, _)
+ if 'AuthorizationException' in str(_):
+ return
+ raise
+ for table_row in table_cursor:
+ table = table_row[0]
+ table_count += 1
+ if not table_regex.search(table):
+ log.debug("skipping database '%s' table '%s', does not match regex '%s'", \
+ database, table, self.table)
+ continue
+ tables.append(table)
+ log.info("%s/%s tables selected for database '%s'", len(tables), table_count, database)
+ for table in tables:
+ try:
+ query = self.query.format(db='`{}`'.format(database),
+ table='`{}`'.format(table))
+ except KeyError as _:
+ if _ == 'db':
+ query = self.query.format(table='`{}`'.format(table))
+ else:
+ raise
+ try:
+ self.execute(conn, database, table, query)
+ self.table_count += 1
+ except Exception as _:
+ if self.ignore_errors:
+ log.error("database '%s' table '%s': %s", database, table, _)
+ continue
+ raise
+
+ @staticmethod
+ def execute(conn, database, table, query):
+ try:
+ log.info(' %s.%s - running %s', database, table, query)
+ with conn.cursor() as query_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ query_cursor.execute(query)
+ for result in query_cursor:
+ print('{db}.{table}\t{result}'.format(db=database, table=table, \
+ result='\t'.join([str(_) for _ in result])))
+ #except (impala.error.OperationalError, impala.error.HiveServer2Error) as _:
+ # log.error(_)
+ except impala.error.ProgrammingError as _:
+ log.error(_)
+ # COMPUTE STATS returns no results
+ if 'Trying to fetch results on an operation with no results' not in str(_):
+ raise
+
+
+if __name__ == '__main__':
+ HiveForEachTable().main()
diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py
index 82b1ab320..ea41f3753 100755
--- a/hive_schemas_csv.py
+++ b/hive_schemas_csv.py
@@ -1,24 +1,24 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
-Connect to a HiveServer2 or Impala daemon and dump all the schemas, tables and columns out in CSV format to stdout
+Connect to HiveServer2 and dump all the schemas, tables and columns out in CSV format to stdout
-In practice Hive is much more reliable for dumping masses of schema
+In practice Hive is much more reliable than Impala for dumping masses of schema
Impala appears faster initially but then slows down more than Hive and hits things query handle errors
under sustained load of extracting large amounts of schema information
@@ -36,152 +36,170 @@
if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will
raise a traceback to tell you to set one (eg. --escapechar='\\')
-Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
https://github.com/cloudera/impyla/issues/286
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
-from __future__ import unicode_literals
+#from __future__ import unicode_literals
-import argparse
import csv
-import logging
import os
-import socket
import sys
-from impala.dbapi import connect
+import impala
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log
+ from hive_impala_cli import HiveImpalaCLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.2.0'
-
-logging.basicConfig()
-log = logging.getLogger(os.path.basename(sys.argv[0]))
-
-host_envs = [
- 'HIVESERVER2_HOST',
- 'HIVE_HOST',
- 'IMPALA_HOST',
- 'HOST'
-]
-
-port_envs = [
- 'HIVESERVER2_PORT',
- 'HIVE_PORT',
- 'IMPALA_PORT',
- 'PORT'
-]
-
-def getenvs(keys, default=None):
- for key in keys:
- value = os.getenv(key)
- if value:
- return value
- return default
-
-def parse_args():
- parser = argparse.ArgumentParser(
- description="Dumps all Hive / Impala schemas, tables, columns and types to CSV format on stdout")
- parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\
- help='HiveServer2 / Impala host ' + \
- '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')')
- parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000),
- help='HiveServer2 / Impala port (default: 10000 if called as hive, ' + \
- '21050 if called as impala, $' + \
- ', $'.join(port_envs) + ')')
- parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)')
- parser.add_argument('-n', '--krb5-service-name', default='hive',
- help='Service principal (default: \'hive\', or \'impala\' if called as impala_schemas_csv.py)')
- parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL')
- # must set type to str otherwise csv module gives this error on Python 2.7:
- # TypeError: "delimiter" must be string, not unicode
- parser.add_argument('-d', '--delimiter', default=',', type=str, help='Delimiter to use (default: ,)')
- parser.add_argument('-Q', '--quotechar', default='"', type=str,
- help='Generate quoted CSV (recommended, default is double quote \'"\')')
- parser.add_argument('-E', '--escapechar', help='Escape char if needed')
- parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode')
- args = parser.parse_args()
-
- if args.verbose:
- log.setLevel(logging.INFO)
-
- if 'impala' in sys.argv[0]:
- if args.krb5_service_name == 'hive':
- log.info('called as impala, setting service principal to impala')
- args.krb5_service_name = 'impala'
- if args.port == 10000:
- log.info('called as impala, setting port to 21050')
- args.port = 21050
- return args
-
-def connect_db(args, database):
- auth_mechanism = None
- if args.kerberos:
- auth_mechanism = 'GSSAPI'
-
- log.info('connecting to %s:%s database %s', args.host, args.port, database)
- return connect(
- host=args.host,
- port=args.port,
- auth_mechanism=auth_mechanism,
- use_ssl=args.ssl,
- #user=user,
- #password=password,
- database=database,
- kerberos_service_name=args.krb5_service_name
- )
-
-def main():
- args = parse_args()
-
- conn = connect_db(args, None)
-
- quoting = csv.QUOTE_ALL
- if args.quotechar == '':
- quoting = csv.QUOTE_NONE
- fieldnames = ['database', 'table', 'column', 'type']
- csv_writer = csv.DictWriter(sys.stdout,
- delimiter=args.delimiter,
- quotechar=args.quotechar,
- escapechar=args.escapechar,
- quoting=quoting,
- fieldnames=fieldnames)
- csv_writer.writeheader()
- log.info('querying databases')
- with conn.cursor() as db_cursor:
- db_cursor.execute('show databases')
- for db_row in db_cursor:
- database = db_row[0]
- log.info('querying tables for database %s', database)
- #db_conn = connect_db(args, database)
- #with db_conn.cursor() as table_cursor:
- with conn.cursor() as table_cursor:
- # doesn't support parameterized query quoting from dbapi spec
- #table_cursor.execute('use %(database)s', {'database': database})
- table_cursor.execute('use {}'.format(database))
- table_cursor.execute('show tables')
- for table_row in table_cursor:
- table = table_row[0]
- log.info('describing table %s', table)
- with conn.cursor() as column_cursor:
- # doesn't support parameterized query quoting from dbapi spec
- #column_cursor.execute('use %(database)s', {'database': database})
- #column_cursor.execute('describe %(table)s', {'table': table})
- column_cursor.execute('use {}'.format(database))
- column_cursor.execute('describe {}'.format(table))
- for column_row in column_cursor:
- column = column_row[0]
- column_type = column_row[1]
- csv_writer.writerow({'database': database,
- 'table': table,
- 'column': column,
- 'type': column_type})
+__version__ = '0.5.1'
+
+
+class HiveSchemasCSV(HiveImpalaCLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveSchemasCSV, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.delimiter = None
+ self.quotechar = None
+ self.escapechar = None
+ self.table_count = 0
+ self.column_count = 0
+ self.ignore_errors = False
+ self.csv_writer = None
+ self.conn = None
+
+ def add_options(self):
+ super(HiveSchemasCSV, self).add_options()
+ # must set type to str otherwise csv module gives this error on Python 2.7:
+ # TypeError: "delimiter" must be string, not unicode
+ # type=str worked with argparse but when integrated with CLI then 'from __future__ import unicode_literals'
+ # breaks this - might break in Python 3 if the impyla module doesn't fix behaviour
+ self.add_opt('-d', '--delimiter', default=',', type=str, help='Delimiter to use (default: ,)')
+ self.add_opt('-Q', '--quotechar', default='"', type=str,
+ help='Generate quoted CSV (recommended, default is double quote \'"\')')
+ self.add_opt('-E', '--escapechar', help='Escape char if needed')
+#
+# ignore tables that fail with errors like:
+#
+# Hive (CDH has MR, no tez):
+#
+# impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long
+#
+# Impala:
+#
+# impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table ''
+# CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table ''
+#
+ self.add_opt('-e', '--ignore-errors', action='store_true',
+ help='Ignore individual table schema listing errors (Impala often has table metadata errors)')
+
+ def process_options(self):
+ super(HiveSchemasCSV, self).process_options()
+ self.delimiter = self.get_opt('delimiter')
+ self.quotechar = self.get_opt('quotechar')
+ self.escapechar = self.get_opt('escapechar')
+ self.ignore_errors = self.get_opt('ignore_errors')
+
+ def run(self):
+
+ self.conn = self.connect('default')
+
+ quoting = csv.QUOTE_ALL
+ if self.quotechar == '':
+ quoting = csv.QUOTE_NONE
+ fieldnames = ['database', 'table', 'column', 'type']
+ self.csv_writer = csv.DictWriter(sys.stdout,
+ delimiter=self.delimiter,
+ quotechar=self.quotechar,
+ escapechar=self.escapechar,
+ quoting=quoting,
+ fieldnames=fieldnames)
+ self.csv_writer.writeheader()
+ log.info('querying databases')
+ databases = []
+ database_count = 0
+ with self.conn.cursor() as db_cursor:
+ db_cursor.execute('show databases')
+ for db_row in db_cursor:
+ database = db_row[0]
+ database_count += 1
+ databases.append(database)
+ log.info('found %s databases', database_count)
+ for database in databases:
+ self.process_database(database)
+ log.info('databases: %s, tables: %s, columns: %s', database_count, self.table_count, self.column_count)
+
+ def process_database(self, database):
+ log.info("querying tables for database '%s'", database)
+ tables = []
+ with self.conn.cursor() as table_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #table_cursor.execute('use %(database)s', {'database': database})
+ table_cursor.execute('use `{}`'.format(database))
+ table_cursor.execute('show tables')
+ for table_row in table_cursor:
+ table = table_row[0]
+ self.table_count += 1
+ tables.append(table)
+ log.info("found %s tables in database '%s'", len(tables), database)
+ for table in tables:
+ try:
+ self.process_table(database, table)
+ except impala.error.HiveServer2Error as _:
+ if self.ignore_errors:
+ log.error("database '%s' table '%s': %s", database, table, _)
+ continue
+ raise
+
+ def process_table(self, database, table):
+ log.info("describing table '%s.%s'", database, table)
+ with self.conn.cursor() as column_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #column_cursor.execute('use %(database)s', {'database': database})
+ #column_cursor.execute('describe %(table)s', {'table': table})
+ column_cursor.execute('use `{}`'.format(database))
+ column_cursor.execute('describe `{}`'.format(table))
+ column_count = 0
+ for column_row in column_cursor:
+ column = column_row[0]
+ column_type = column_row[1]
+ column_count += 1
+ self.csv_writer.writerow({'database': database,
+ 'table': table,
+ 'column': column,
+ 'type': column_type})
+ log.info("found %s columns in table '%s.%s'", column_count, database, table)
+ self.column_count += column_count
if __name__ == '__main__':
- main()
+ HiveSchemasCSV().main()
diff --git a/hive_tables_column_counts.py b/hive_tables_column_counts.py
new file mode 100755
index 000000000..d6cb4ce68
--- /dev/null
+++ b/hive_tables_column_counts.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and count the number of columns for each table in each database,
+or only those matching given db / table regexes
+
+You can also get this from the schemas.csv output generated by hive_schemas_csv.py, eg.
+
+ tail -n +2 hive_schemas.csv | cut -d, -f1,2 | sed 's/"//g; s/,/./' | sort | uniq -c | sort -k1nr
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class HiveTablesColumnCounts(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesColumnCounts, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.query = 'placeholder' # not needed, here to suppress --query CLI option
+ self.database = None
+ self.table = None
+ self.ignore_errors = False
+
+ # discarding last param query
+ def execute(self, conn, database, table, query):
+ column_count = 0
+ log.info("describing table '%s.%s'", database, table)
+ with conn.cursor() as column_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #column_cursor.execute('use %(database)s', {'database': database})
+ #column_cursor.execute('describe %(table)s', {'table': table})
+ column_cursor.execute('use `{}`'.format(database))
+ # don't use desc here, Impala doesn't support it and would break subclass
+ column_cursor.execute('describe `{}`'.format(table))
+ for _ in column_cursor:
+ #column = _[0]
+ #column_type = _[1]
+ column_count += 1
+ print('{db}.{table}\t{column_count}'.format(db=database, table=table, column_count=column_count))
+
+
+if __name__ == '__main__':
+ HiveTablesColumnCounts().main()
diff --git a/hive_tables_list.py b/hive_tables_list.py
new file mode 100755
index 000000000..64675c61f
--- /dev/null
+++ b/hive_tables_list.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and list all databases and tables
+
+TSV Output format:
+
+
+
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+
+class HiveTablesList(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesList, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.query = 'placeholder' # here merely to suppress --query CLI option
+
+ def execute(self, conn, database, table, query):
+ print('{}\t{}'.format(database, table))
+
+
+if __name__ == '__main__':
+ HiveTablesList().main()
diff --git a/hive_tables_locations.py b/hive_tables_locations.py
new file mode 100755
index 000000000..403e07443
--- /dev/null
+++ b/hive_tables_locations.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and list the locations of all tables in all databases,
+or only those matching given db / table regexes
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_metadata import HiveTablesMetadata
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class HiveTablesLocations(HiveTablesMetadata):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesLocations, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.field = 'Location'
+
+
+if __name__ == '__main__':
+ HiveTablesLocations().main()
diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py
new file mode 100755
index 000000000..d15d62feb
--- /dev/null
+++ b/hive_tables_metadata.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and print the first matching DDL metadata field (eg. 'Location')
+for each table in each database, or only those matching given db / table regexes
+
+Examples (fields are case sensitive regex and return N/A without match):
+
+./hive_tables_metadata.py --field Location ...
+./hive_tables_metadata.py --field SerDe ...
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import re
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_regex
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.1'
+
+
+class HiveTablesMetadata(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesMetadata, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.query = 'describe formatted {table}'
+ self.field = None
+
+ def add_options(self):
+ # Python 2.x
+ super(HiveTablesMetadata, self).add_options()
+ # Python 3.x
+ # super().__init__()
+ if self.field is None:
+ self.add_opt('-f', '--field', help='Table DDL metadata field to return for each table (required)')
+
+ def process_options(self):
+ # Python 2.x
+ super(HiveTablesMetadata, self).process_options()
+ # Python 3.x
+ # super().__init__()
+ if self.field is None:
+ self.field = self.get_opt('field')
+ if not self.field:
+ self.usage('--field not specified')
+ validate_regex(self.field, 'field')
+ self.field = re.compile(self.field)
+
+ # discard last param query and construct our own based on the table DDL of cols
+ def execute(self, conn, database, table, query):
+ log.info("describing table '%s.%s'", database, table)
+ field = 'N/A'
+ with conn.cursor() as table_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #table_cursor.execute('use %(database)s', {'database': database})
+ #table_cursor.execute('describe %(table)s', {'table': table})
+ table_cursor.execute('use `{}`'.format(database))
+ table_cursor.execute(query.format(table=table))
+ for row in table_cursor:
+ if self.field.search(row[0]):
+ field = row[1]
+ break
+ print('{db}.{table}\t{field}'.format(db=database, table=table, field=field))
+
+
+if __name__ == '__main__':
+ HiveTablesMetadata().main()
diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py
new file mode 100755
index 000000000..207fcba3f
--- /dev/null
+++ b/hive_tables_null_columns.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and find tables with columns containing only NULLs
+for all tables in all databases, or only those matching given db / table regexes
+
+Describes each table, constructs a complex query to check each column individually for containing only NULLs,
+and prints out each tables' count of total columns containing only NULLs as well as the list of offending columns
+
+Useful for catching problems with data quality or subtle ETL bugs
+
+Rewrite of a Perl version from 2014 from my DevOps Perl Tools repo
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class HiveTablesNullColumns(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesNullColumns, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.query = 'placeholder' # constructed later dynamically per table, here to suppress --query CLI option
+ self.database = None
+ self.table = None
+ #self.partition = None
+ self.ignore_errors = False
+
+ # discard last param query and construct our own based on the table DDL of cols
+ def execute(self, conn, database, table, _query):
+ sum_part = ''
+ columns = []
+ log.info("describing table '%s.%s'", database, table)
+ with conn.cursor() as column_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #column_cursor.execute('use %(database)s', {'database': database})
+ #column_cursor.execute('describe %(table)s', {'table': table})
+ column_cursor.execute('use `{}`'.format(database))
+ column_cursor.execute('describe `{}`'.format(table))
+ for column_row in column_cursor:
+ column = column_row[0]
+ #column_type = column_row[1]
+ columns.append(column)
+ sum_part = ', '.join(
+ ['IF(SUM(IF(`{col}` IS NULL, 1, 0)) = COUNT(*), 1, 0) as `{col}`'.format(col=column) \
+ for column in columns]
+ )
+ query = "SELECT {sum_part} FROM `{db}`.`{table}` WHERE `"\
+ .format(sum_part=sum_part, db=database, table=table) + \
+ "` IS NULL OR `".join(columns) + "` IS NULL"
+ self.check_table_for_nulls(conn, database, table, columns, query)
+
+ @staticmethod
+ def check_table_for_nulls(conn, database, table, columns, query):
+ with conn.cursor() as table_cursor:
+ log.debug('executing query: %s', query)
+ table_cursor.execute(query)
+ cols_with_nulls = []
+ for result in table_cursor:
+ # tuple of ints (0, 0, 0, .... N) - one per column
+ for index in range(len(list(result))):
+ col_result = result[index]
+ if col_result > 0:
+ cols_with_nulls.append(columns[index])
+ num_cols = len(cols_with_nulls)
+ total_cols = len(columns)
+ if cols_with_nulls:
+ print('WARNING: {db}.{table} has {num}/{total} columns with only NULLs: {cols}'\
+ .format(db=database,
+ table=table,
+ num=num_cols,
+ total=total_cols,
+ cols=', '.join(sorted(cols_with_nulls))
+ )
+ )
+ else:
+ print('OK: {db}.{table} has {num}/{total} columns with only nulls'\
+ .format(db=database, table=table, num=num_cols, total=total_cols))
+
+
+if __name__ == '__main__':
+ HiveTablesNullColumns().main()
diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py
new file mode 100755
index 000000000..f1a5752f1
--- /dev/null
+++ b/hive_tables_null_rows.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and count number of rows with only NULLs in all columns
+for each table in each database, or only those matching given db / table regexes
+
+Useful for catching problems with data quality or subtle ETL bugs
+
+Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.6.0'
+
+
+class HiveTablesNullRows(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesNullRows, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.query = 'placeholder' # constructed later dynamically per table, here to suppress --query CLI option
+ self.database = None
+ self.table = None
+ #self.partition = None
+ self.ignore_errors = False
+
+ # discard last param query and construct our own based on the table DDL of cols
+ def execute(self, conn, database, table, _):
+ columns = []
+ log.info("describing table '%s.%s'", database, table)
+ with conn.cursor() as column_cursor:
+ # impala library doesn't support parameterized query quoting from dbapi spec
+ #column_cursor.execute('use %(database)s', {'database': database})
+ #column_cursor.execute('describe %(table)s', {'table': table})
+ column_cursor.execute('use `{}`'.format(database))
+ column_cursor.execute('describe `{}`'.format(table))
+ for column_row in column_cursor:
+ column = column_row[0]
+ #column_type = column_row[1]
+ columns.append(column)
+ query = self.generate_sql(database, table, columns)
+ with conn.cursor() as table_cursor:
+ log.debug('executing query: %s', query)
+ table_cursor.execute(query)
+ for result in table_cursor:
+ count = result[0]
+ print('{db}.{table}\t{count}'.format(db=database, table=table, count=count))
+
+ @staticmethod
+ def generate_sql(database, table, columns):
+ sql = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\
+ .format(db=database, table=table) + \
+ "` IS NULL AND `".join(columns) + "` IS NULL"
+ return sql
+
+
+if __name__ == '__main__':
+ HiveTablesNullRows().main()
diff --git a/hive_tables_row_column_counts.py b/hive_tables_row_column_counts.py
new file mode 100755
index 000000000..c2a5ec7e9
--- /dev/null
+++ b/hive_tables_row_column_counts.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and count the number of rows and columns for each table
+in each database, or only those matching given db / table regexes
+
+Output format:
+
+ .
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class HiveTablesRowColumnCounts(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesRowColumnCounts, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.query = 'placeholder' # not needed, here to suppress --query CLI option
+ self.database = None
+ self.table = None
+ self.ignore_errors = False
+
+ # discarding last param query
+ def execute(self, conn, database, table, query):
+ row_count = None
+ column_count = 0
+ log.info("describing table '%s.%s'", database, table)
+ with conn.cursor() as cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #cursor.execute('use %(database)s', {'database': database})
+ #cursor.execute('describe %(table)s', {'table': table})
+ cursor.execute('use `{}`'.format(database))
+ # don't use desc here, Impala doesn't support it and would break subclass
+ cursor.execute('describe `{}`'.format(table))
+ for _ in cursor:
+ #column = _[0]
+ #column_type = _[1]
+ column_count += 1
+ log.info("running SELECT COUNT(*) FROM `%s`.`%s`", database, table)
+ # doesn't support parameterized query quoting from dbapi spec
+ cursor.execute('SELECT COUNT(*) FROM `{db}`.`{table}`'.format(db=database, table=table))
+ for result in cursor:
+ assert row_count is None
+ row_count = result[0]
+ print('{db}.{table}\t{row_count}\t{column_count}'\
+ .format(db=database,
+ table=table,
+ row_count=row_count,
+ column_count=column_count))
+
+
+if __name__ == '__main__':
+ HiveTablesRowColumnCounts().main()
diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py
new file mode 100755
index 000000000..27aaa1a9c
--- /dev/null
+++ b/hive_tables_row_counts.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and get rows counts for all tables in all databases,
+or only those matching given db / table / partition value regexes
+
+Useful for reconciliations between clusters after migrations
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import re
+import sys
+import impala
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_regex
+ from hive_impala_cli import HiveImpalaCLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.6.1'
+
+
+class HiveTablesRowCounts(HiveImpalaCLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesRowCounts, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.database = None
+ self.table = None
+ self.partition = None
+ self.ignore_errors = False
+ self.table_count = 0
+
+ def add_options(self):
+ super(HiveTablesRowCounts, self).add_options()
+ self.add_opt('-d', '--database', default='.*', help='Database regex (default: .*)')
+ self.add_opt('-t', '--table', default='.*', help='Table regex (default: .*)')
+ self.add_opt('-p', '--partition', default='.*', help='Partition regex (default: .*)')
+#
+# ignore tables that fail with errors like:
+#
+# Hive (CDH has MR, no tez):
+#
+# impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long
+#
+# Impala:
+#
+# impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table ''
+# CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table ''
+#
+ self.add_opt('-e', '--ignore-errors', action='store_true', help='Ignore individual table errors and continue')
+
+ def process_options(self):
+ super(HiveTablesRowCounts, self).process_options()
+ self.database = self.get_opt('database')
+ self.table = self.get_opt('table')
+ self.partition = self.get_opt('partition')
+ self.ignore_errors = self.get_opt('ignore_errors')
+ validate_regex(self.database, 'database')
+ validate_regex(self.table, 'table')
+ validate_regex(self.partition, 'partition')
+
+ def run(self):
+ database_regex = re.compile(self.database, re.I)
+ table_regex = re.compile(self.table, re.I)
+ partition_regex = re.compile(self.partition, re.I)
+ conn = self.connect('default')
+ log.info('querying databases')
+ # collecting in local list because long time iteration results in
+ # impala.error.HiveServer2Error: Invalid query handle
+ databases = []
+ database_count = 0
+ with conn.cursor() as db_cursor:
+ db_cursor.execute('show databases')
+ for db_row in db_cursor:
+ database = db_row[0]
+ database_count += 1
+ if not database_regex.search(database):
+ log.debug("skipping database '%s', does not match regex '%s'", database, self.database)
+ continue
+ databases.append(database)
+ log.info('%s/%s databases selected', len(databases), database_count)
+ for database in databases:
+ self.process_database(conn, database, table_regex, partition_regex)
+ log.info('processed %s databases, %s tables', database_count, self.table_count)
+
+ def process_database(self, conn, database, table_regex, partition_regex):
+ tables = []
+ table_count = 0
+ log.info("querying tables for database '%s'", database)
+ with conn.cursor() as table_cursor:
+ try:
+ # doesn't support parameterized query quoting from dbapi spec
+ #table_cursor.execute('use %(database)s', {'database': database})
+ table_cursor.execute('use `{}`'.format(database))
+ table_cursor.execute('show tables')
+ except impala.error.HiveServer2Error as _:
+ log.error(_)
+ if 'AuthorizationException' in str(_):
+ return
+ raise
+ for table_row in table_cursor:
+ table = table_row[0]
+ table_count += 1
+ if not table_regex.search(table):
+ log.debug("skipping database '%s' table '%s', does not match regex '%s'", \
+ database, table, self.table)
+ continue
+ tables.append(table)
+ log.info("%s/%s tables selected for database '%s'", len(tables), table_count, database)
+ for table in tables:
+ try:
+ self.get_row_counts(conn, database, table, partition_regex)
+ self.table_count += 1
+ except Exception as _:
+ # invalid query handle and similar errors happen at higher level
+ # as they are not query specific, will not be caught here so still error out
+ if self.ignore_errors:
+ log.error("database '%s' table '%s': %s", database, table, _)
+ continue
+ raise
+
+ def get_row_counts(self, conn, database, table, partition_regex):
+ log.info("getting partitions for database '%s' table '%s'", database, table)
+ with conn.cursor() as partition_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ partition_cursor.execute('use `{db}`'.format(db=database))
+ try:
+ partition_cursor.execute('show partitions `{table}`'.format(table=table))
+ for partitions_row in partition_cursor:
+ partition_key = partitions_row[0]
+ partition_value = partitions_row[1]
+ if not partition_regex.match(partition_value):
+ log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " +
+ "value does not match regex '%s'",
+ database,
+ table,
+ partition_key,
+ partition_value,
+ self.partition)
+ continue
+ # doesn't support parameterized query quoting from dbapi spec
+ partition_cursor.execute('SELECT COUNT(*) FROM `{db}`.`{table}` WHERE `{key}`={value}'\
+ .format(db=database, table=table, key=partition_key, value=partition_value))
+ for result in partition_cursor:
+ row_count = result[0]
+ print('{db}.{table}.{key}={value}\t{row_count}'.format(\
+ db=database,
+ table=table,
+ key=partition_key,
+ value=partition_value,
+ row_count=row_count))
+ except (impala.error.OperationalError, impala.error.HiveServer2Error) as _:
+ # Hive impala.error.HiveServer2Error: is not a partitioned table
+ # Impala impala.error.HiveServer2Error: Table is not partitioned
+ if 'is not a partitioned table' not in str(_) and \
+ 'Table is not partitioned' not in str(_):
+ raise
+ log.info("no partitions found for database '%s' table '%s', getting row counts for whole table",
+ database, table)
+ with conn.cursor() as table_cursor:
+ log.info("running SELECT COUNT(*) FROM `%s`.`%s`", database, table)
+ # doesn't support parameterized query quoting from dbapi spec
+ table_cursor.execute('SELECT COUNT(*) FROM `{db}`.`{table}`'.format(db=database, table=table))
+ for result in table_cursor:
+ row_count = result[0]
+ print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count))
+
+
+if __name__ == '__main__':
+ HiveTablesRowCounts().main()
diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py
new file mode 100755
index 000000000..8ea7e6b75
--- /dev/null
+++ b/hive_tables_row_counts_any_nulls.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and count number of rows with NULL in any column
+for each table in each database, or only those matching given db / table regexes
+
+Useful for catching problems with data quality or subtle ETL bugs
+
+Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_null_rows import HiveTablesNullRows
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.6.0'
+
+
+class HiveTablesRowsWithNulls(HiveTablesNullRows):
+
+ @staticmethod
+ def generate_sql(database, table, columns):
+ # impala library doesn't support parameterized query quoting from dbapi spec
+ sql = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\
+ .format(db=database, table=table) + \
+ "` IS NULL OR `".join(columns) + "` IS NULL"
+ return sql
+
+
+if __name__ == '__main__':
+ HiveTablesRowsWithNulls().main()
diff --git a/impala_compute_table_stats.py b/impala_compute_table_stats.py
new file mode 100755
index 000000000..04d462e16
--- /dev/null
+++ b/impala_compute_table_stats.py
@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to a Impalad and compute optimization statistics on all tables,
+or only those matching given db / table / partition value regexes
+
+Tested on CDH 5.10, Impala 1.1.0 with Kerberos
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
+import logging
+import os
+import re
+import socket
+import sys
+# pylint: disable=import-error
+from impala.dbapi import connect
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+logging.basicConfig()
+log = logging.getLogger(os.path.basename(sys.argv[0]))
+
+host_envs = [
+ 'IMPALA_HOST',
+ 'HOST'
+]
+
+port_envs = [
+ 'IMPALA_PORT',
+ 'PORT'
+]
+
+def getenvs(keys, default=None):
+ for key in keys:
+ value = os.getenv(key)
+ if value:
+ return value
+ return default
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description="Computes statistics on all Impala tables/partitions matching database / table / partition regexes")
+ parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\
+ help='Impalad host ' + \
+ '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')')
+ parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 21050),
+ help='Impalad port (default: 21050, ' + ', $'.join(port_envs) + ')')
+ parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)')
+ parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)')
+ parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)')
+ parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)')
+ parser.add_argument('-n', '--krb5-service-name', default='hive',
+ help='Service principal (default: \'hive\')')
+ parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL')
+ parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode')
+ args = parser.parse_args()
+
+ if args.verbose:
+ log.setLevel(logging.INFO)
+
+ return args
+
+def connect_db(args, database):
+ auth_mechanism = None
+ if args.kerberos:
+ auth_mechanism = 'GSSAPI'
+
+ log.info('connecting to %s:%s database %s', args.host, args.port, database)
+ return connect(
+ host=args.host,
+ port=args.port,
+ auth_mechanism=auth_mechanism,
+ use_ssl=args.ssl,
+ #user=user,
+ #password=password,
+ database=database,
+ kerberos_service_name=args.krb5_service_name
+ )
+
+def main():
+ args = parse_args()
+
+ try:
+ database_regex = re.compile(args.database, re.I)
+ table_regex = re.compile(args.table, re.I)
+ partition_regex = re.compile(args.partition, re.I)
+ except re.error as _:
+ log.error('error in provided regex: %s', _)
+ sys.exit(3)
+
+ conn = connect_db(args, None)
+
+ log.info('querying databases')
+ with conn.cursor() as db_cursor:
+ db_cursor.execute('show databases')
+ for db_row in db_cursor:
+ database = db_row[0]
+ if not database_regex.search(database):
+ log.debug("skipping database '%s', does not match regex '%s'", database, args.database)
+ continue
+ log.info('querying tables for database %s', database)
+ #db_conn = connect_db(args, database)
+ #with db_conn.cursor() as table_cursor:
+ with conn.cursor() as table_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #table_cursor.execute('use %(database)s', {'database': database})
+ table_cursor.execute('use {}'.format(database))
+ table_cursor.execute('show tables')
+ for table_row in table_cursor:
+ table = table_row[0]
+ if not table_regex.search(table):
+ log.debug("skipping database '%s' table '%s', does not match regex '%s'", \
+ database, table, args.table)
+ continue
+ compute_table_stats(conn, args, database, table, partition_regex)
+
+def compute_table_stats(conn, args, database, table, partition_regex):
+ log.info("getting partitions for database '%s' table '%s'", database, table)
+ partitions_found = False
+ with conn.cursor() as partition_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ partition_cursor.execute('use {}'.format(database))
+ partition_cursor.execute('show partitions {}'.format(database))
+ for partitions_row in partition_cursor:
+ partition_key = partitions_row[0]
+ partition_value = partitions_row[1]
+ partitions_found = True
+ if not partition_regex.match(partition_value):
+ # pylint: disable=logging-not-lazy
+ log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " +
+ "value does not match regex '%s'",
+ database,
+ table,
+ partition_key,
+ partition_value,
+ args.partition)
+ continue
+ # doesn't support parameterized query quoting from dbapi spec
+ partition_cursor.execute('COMPUTE INCREMENTAL STATS {db}.{table} PARTITION({key}={value})'\
+ .format(db=database, table=table, key=partition_key, value=partition_value))
+ if not partitions_found:
+ log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table)
+ with conn.cursor() as table_cursor:
+ log.info("running compute stats on table '%s'", table)
+ # doesn't support parameterized query quoting from dbapi spec
+ table_cursor.execute('COMPUTE STATS {db}.{table}'.format(db=database, table=table))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/impala_foreach_table.py b/impala_foreach_table.py
new file mode 100755
index 000000000..f4e324ac9
--- /dev/null
+++ b/impala_foreach_table.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and execute a query for each table in each database,
+or only those matching given db / table regexes
+
+Useful for getting row counts of all tables or analyzing tables:
+
+eg.
+
+impala_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}'
+impala_foreach_table.py --query 'COMPUTE STATS {table}'
+
+or just for today's partition:
+
+impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))"
+
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaForEachTable(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaForEachTable, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaForEachTable().main()
diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py
deleted file mode 120000
index baed3c963..000000000
--- a/impala_schemas_csv.py
+++ /dev/null
@@ -1 +0,0 @@
-hive_schemas_csv.py
\ No newline at end of file
diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py
new file mode 100755
index 000000000..0ac8491dd
--- /dev/null
+++ b/impala_schemas_csv.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and dump all the schemas, tables and columns out in CSV format to stdout
+
+In practice Hive is much more reliable than Impala for dumping masses of schema (see adjacent hive_schemas_csv.py)
+
+Impala appears faster initially but then slows down more than Hive and hits things query handle errors
+under sustained load of extracting large amounts of schema information
+
+There is also a risk that Impala's metadata may be out of date, so Hive is strongly preferred for this
+
+
+CSV format:
+
+database,table,column,type
+
+
+I recommend generating quoted csv because you may encounter Hive data types such as decimal(15,2)
+which would cause incorrect field splitting, you can disable by setting --quotechar='' to blank but
+if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will
+raise a traceback to tell you to set one (eg. --escapechar='\\')
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+from hive_schemas_csv import HiveSchemasCSV
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class ImpalaSchemasCSV(HiveSchemasCSV):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaSchemasCSV, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaSchemasCSV().main()
diff --git a/impala_tables_column_counts.py b/impala_tables_column_counts.py
new file mode 100755
index 000000000..e62027542
--- /dev/null
+++ b/impala_tables_column_counts.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and count the number of columns for each table in each database,
+or only those matching given db / table regexes
+
+You can also get this from the schemas.csv output generated by impala_schemas_csv.py, eg.
+
+ tail -n +2 impala_schemas.csv | cut -d, -f1,2 | sed 's/"//g; s/,/./' | sort | uniq -c | sort -k1nr
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_column_counts import HiveTablesColumnCounts
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesColumnCounts(HiveTablesColumnCounts):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesColumnCounts, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesColumnCounts().main()
diff --git a/impala_tables_list.py b/impala_tables_list.py
new file mode 100755
index 000000000..a45194ad3
--- /dev/null
+++ b/impala_tables_list.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and list all databases and tables
+
+TSV Output format:
+
+
+
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_list import HiveTablesList
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+
+class ImpalaTablesList(HiveTablesList):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesList, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesList().main()
diff --git a/impala_tables_locations.py b/impala_tables_locations.py
new file mode 100755
index 000000000..383bb1154
--- /dev/null
+++ b/impala_tables_locations.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and list the locations of all tables in all databases,
+or only those matching given db / table regexes
+
+Caveats:
+
+ Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't
+
+ Impala is faster than Hive for the first ~1000 tables but then slows down
+ so if you have a lot of tables I recommend you use the Hive version of this instead
+ eg. by ~1900 tables the Hive version will overtake the Impala version and
+ for thousands of tables Impala actuallys runs 1.5 - 2x slower than the Hive version overall
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_locations import HiveTablesLocations
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesLocations(HiveTablesLocations):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesLocations, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesLocations().main()
diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py
new file mode 100755
index 000000000..501b82c93
--- /dev/null
+++ b/impala_tables_metadata.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and print the first matching DDL metadata field (eg. 'Location')
+for each table in each database, or only those matching given db / table regexes
+
+Examples (fields are case sensitive regex and return N/A without match):
+
+./impala_tables_metadata.py --field Location ...
+./impala_tables_metadata.py --field SerDe ...
+
+Caveats:
+
+ Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't
+
+ Impala is faster than Hive for the first ~1000 tables but then slows down
+ so if you have a lot of tables I recommend you use the Hive version of this instead
+ eg. by ~1900 tables the Hive version will overtake the Impala version and
+ for thousands of tables Impala actuallys runs 1.5 - 2x slower than the Hive version overall
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_metadata import HiveTablesMetadata
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesMetadata(HiveTablesMetadata):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesMetadata, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesMetadata().main()
diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py
new file mode 100755
index 000000000..2b70caf3f
--- /dev/null
+++ b/impala_tables_null_columns.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and find tables with columns containing only NULLs
+for all tables in all databases, or only those matching given db / table regexes
+
+Describes each table, constructs a complex query to check each column individually for containing only NULLs,
+and prints out each tables' count of total columns containing only NULLs as well as the list of offending columns
+
+Useful for catching problems with data quality or subtle ETL bugs
+
+Rewrite of a Perl version from 2014 from my DevOps Perl Tools repo
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_null_columns import HiveTablesNullColumns
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesNullColumns(HiveTablesNullColumns):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesNullColumns, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesNullColumns().main()
diff --git a/impala_tables_null_rows.py b/impala_tables_null_rows.py
new file mode 100755
index 000000000..4fea8a82e
--- /dev/null
+++ b/impala_tables_null_rows.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and count number of rows with only NULLs in all columns
+for each tables in each database, or only those matching given db / table regexes
+
+Useful for catching problems with data quality or subtle ETL bugs
+
+Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_null_rows import HiveTablesNullRows
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesNullRows(HiveTablesNullRows):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesNullRows, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesNullRows().main()
diff --git a/impala_tables_row_column_counts.py b/impala_tables_row_column_counts.py
new file mode 100755
index 000000000..8cf4d6b82
--- /dev/null
+++ b/impala_tables_row_column_counts.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and count the number of rows and columns for each table
+in each database, or only those matching given db / table regexes
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_row_column_counts import HiveTablesRowColumnCounts
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesRowColumnCounts(HiveTablesRowColumnCounts):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesRowColumnCounts, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesRowColumnCounts().main()
diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py
new file mode 100755
index 000000000..decc2982c
--- /dev/null
+++ b/impala_tables_row_counts.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and get rows counts for all tables in all databases,
+or only those matching given db / table / partition value regexes
+
+Useful for reconciliations between clusters after migrations
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_row_counts import HiveTablesRowCounts
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class ImpalaTablesRowCounts(HiveTablesRowCounts):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesRowCounts, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesRowCounts().main()
diff --git a/impala_tables_row_counts_any_nulls.py b/impala_tables_row_counts_any_nulls.py
new file mode 100755
index 000000000..d7b49a3b2
--- /dev/null
+++ b/impala_tables_row_counts_any_nulls.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and count number of rows with NULL in any column
+for each table in each database, or only those matching given db / table regexes
+
+Useful for catching problems with data quality or subtle ETL bugs
+
+Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_row_counts_any_nulls import HiveTablesRowsWithNulls
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesRowsWithNulls(HiveTablesRowsWithNulls):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesRowsWithNulls, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesRowsWithNulls().main()
diff --git a/ipython_notebook_pyspark.py b/ipython_notebook_pyspark.py
index 013d61ff2..4c4505015 100755
--- a/ipython_notebook_pyspark.py
+++ b/ipython_notebook_pyspark.py
@@ -1,9 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
#
# Author: Hari Sekhon
# Date: 6/8/2014
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
@@ -93,7 +93,7 @@
# TODO: rewrite from here as a CLI class, not using globals
if len(sys.argv) > 1:
- printerr("""Hari Sekhon - https://github.com/harisekhon/devops-python-tools
+ printerr("""Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools
usage: %s
diff --git a/json_docs_to_bulk_multiline.py b/json_docs_to_bulk_multiline.py
index 98f4d4db9..eb49bcbeb 100755
--- a/json_docs_to_bulk_multiline.py
+++ b/json_docs_to_bulk_multiline.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2017-07-28 17:08:47 +0200 (Fri, 28 Jul 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/json_to_xml.py b/json_to_xml.py
index ca2afcafb..54c0d6395 100755
--- a/json_to_xml.py
+++ b/json_to_xml.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-01-15 00:07:09 +0000 (Fri, 15 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: GNU GPL version 2 (this file only), rest of this repo is licensed as per the adjacent LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# http://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -52,7 +52,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.1'
+__version__ = '0.2.0'
class JsonToXml(CLI):
@@ -110,7 +110,7 @@ def process_file(self, filepath):
if filepath == '-':
filepath = ''
if filepath == '':
- self.json_to_xml(sys.stdin.read())
+ print(self.json_to_xml(sys.stdin.read()))
else:
with open(filepath) as _:
content = _.read()
diff --git a/json_to_yaml.py b/json_to_yaml.py
new file mode 100755
index 000000000..674f6729f
--- /dev/null
+++ b/json_to_yaml.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-12-19 17:54:21 +0000 (Thu, 19 Dec 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tool to convert JSON to YAML
+
+Reads any given files as JSON and prints the equivalent YAML to stdout for piping or redirecting to a file.
+
+Directories if given are detected and recursed, processing all files in the directory tree ending in a .json suffix.
+
+Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from
+standard input.
+
+Written to convert old AWS CloudFormation json templates to yaml
+
+See also:
+
+
+ json2yaml.sh - https://github.com/HariSekhon/DevOps-Bash-tools
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import json
+import os
+import re
+import sys
+import yaml
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import die, ERRORS, log, log_option
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.2.0'
+
+
+class JsonToYaml(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(JsonToYaml, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.re_json_suffix = re.compile(r'.*\.json$', re.I)
+
+ @staticmethod
+ def json_to_yaml(content, filepath=None):
+ try:
+ _ = json.loads(content)
+ except (KeyError, ValueError) as _:
+ file_detail = ''
+ if filepath is not None:
+ file_detail = ' in file \'{0}\''.format(filepath)
+ die("Failed to parse JSON{0}: {1}".format(file_detail, _))
+ return yaml.safe_dump(_)
+
+ def run(self):
+ if not self.args:
+ self.args.append('-')
+ for arg in self.args:
+ if arg == '-':
+ continue
+ if not os.path.exists(arg):
+ print("'%s' not found" % arg)
+ sys.exit(ERRORS['WARNING'])
+ if os.path.isfile(arg):
+ log_option('file', arg)
+ elif os.path.isdir(arg):
+ log_option('directory', arg)
+ else:
+ die("path '%s' could not be determined as either a file or directory" % arg)
+ for arg in self.args:
+ self.process_path(arg)
+
+ def process_path(self, path):
+ if path == '-' or os.path.isfile(path):
+ self.process_file(path)
+ elif os.path.isdir(path):
+ for root, _, files in os.walk(path):
+ for filename in files:
+ filepath = os.path.join(root, filename)
+ if self.re_json_suffix.match(filepath):
+ self.process_file(filepath)
+ else:
+ die("failed to determine if path '%s' is a file or directory" % path)
+
+ def process_file(self, filepath):
+ log.debug('processing filepath \'%s\'', filepath)
+ if filepath == '-':
+ filepath = ''
+ if filepath == '':
+ print(self.json_to_yaml(sys.stdin.read()))
+ else:
+ with open(filepath) as _:
+ content = _.read()
+ print('---')
+ print(self.json_to_yaml(content, filepath=filepath))
+
+
+if __name__ == '__main__':
+ JsonToYaml().main()
diff --git a/jython_autoinstall.exp b/jython_autoinstall.exp
index 1230a03a1..342488e6a 100755
--- a/jython_autoinstall.exp
+++ b/jython_autoinstall.exp
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: Fri Jun 17 15:12:17 2016 +0100
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set force_conservative 0 ;# set to 1 to force conservative mode even if
diff --git a/jython_install.sh b/jython_install.sh
index b56c94cac..4d3f420ee 100755
--- a/jython_install.sh
+++ b/jython_install.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-08-01 10:17:55 +0100 (Mon, 01 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/kics.config b/kics.config
new file mode 100644
index 000000000..51ac5712e
--- /dev/null
+++ b/kics.config
@@ -0,0 +1,44 @@
+#
+# Author: Hari Sekhon
+# Date: 2023-05-05 18:05:53 +0100 (Fri, 05 May 2023)
+#
+# vim:ts=2:sts=2:sw=2:et:filetype=yaml
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# K i c s C o n f i g
+# ============================================================================ #
+
+# https://github.com/Checkmarx/kics/blob/master/docs/configuration-file.md
+
+---
+#path: assets/iac_samples
+verbose: true
+log-file: true
+#type:
+# - Dockerfile
+# - Kubernetes
+#queries-path: "assets/queries"
+exclude-paths:
+ # ignore submodules - handle them in the source repos only
+ - bash-tools/
+ - github-actions/
+ - haproxy-configs/
+ - jenkins/
+ - kubernetes-templates/
+ - lib/
+ - pylib/
+ - spotify-tools/
+ - sql/
+ - sql-keywords/
+ - templates/
+ - terraform-templates/
+#output-path: "results"
diff --git a/lib/__init__.py b/lib/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/lib/hive_impala_cli.py b/lib/hive_impala_cli.py
new file mode 100755
index 000000000..958ccec69
--- /dev/null
+++ b/lib/hive_impala_cli.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import socket
+import sys
+from impala.dbapi import connect
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_host, validate_port
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class HiveImpalaCLI(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveImpalaCLI, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.name = ['HiveServer2', 'Hive']
+ self.host = None
+ self.port = None
+ self.default_host = socket.getfqdn()
+ self.default_port = 10000
+ self.default_service_name = 'hive'
+ self.kerberos = False
+ self.krb5_service_name = self.default_service_name
+ self.ssl = False
+ self.verbose_default = 1
+ #self.timeout_default = 86400
+ self.timeout_default = None
+ if 'impala' in sys.argv[0]:
+ self.name = 'Impala'
+ self.default_port = 21050
+ self.default_service_name = 'impala'
+
+ def add_options(self):
+ super(HiveImpalaCLI, self).add_options()
+ self.add_hostoption()
+ self.add_opt('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)')
+ self.add_opt('-n', '--krb5-service-name', default=self.default_service_name,
+ help='Service principal (default: {})'.format(self.default_service_name))
+ self.add_opt('-S', '--ssl', action='store_true', help='Use SSL')
+
+ def process_options(self):
+ super(HiveImpalaCLI, self).process_options()
+ self.host = self.get_opt('host')
+ self.port = self.get_opt('port')
+ validate_host(self.host)
+ validate_port(self.port)
+ self.port = int(self.port)
+ self.kerberos = self.get_opt('kerberos')
+ self.krb5_service_name = self.get_opt('krb5_service_name')
+ self.ssl = self.get_opt('ssl')
+
+ def connect(self, database):
+ auth_mechanism = None
+ if self.kerberos:
+ auth_mechanism = 'GSSAPI'
+ log.debug('kerberos enabled')
+ log.debug('krb5 remote service principal name = %s', self.krb5_service_name)
+ if self.ssl:
+ log.debug('ssl enabled')
+
+ log.info('connecting to %s:%s database %s', self.host, self.port, database)
+ return connect(
+ host=self.host,
+ port=self.port,
+ auth_mechanism=auth_mechanism,
+ use_ssl=self.ssl,
+ #user=user,
+ #password=password,
+ database=database,
+ kerberos_service_name=self.krb5_service_name
+ )
diff --git a/lib/postgres_cli.py b/lib/postgres_cli.py
new file mode 100755
index 000000000..e84416d98
--- /dev/null
+++ b/lib/postgres_cli.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-03-12 17:39:57 +0000 (Thu, 12 Mar 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import socket
+import sys
+import psycopg2
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_host, validate_port, validate_user, validate_password
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+
+class PostgreSQLCLI(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(PostgreSQLCLI, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.name = ['PostgreSQL', 'Postgres', 'PG']
+ self.host = None
+ self.port = None
+ self.default_host = socket.getfqdn()
+ self.default_port = 5432
+ self.user = None
+ self.password = None
+ #self.ssl = False
+ self.verbose_default = 1
+ self.timeout_default = None
+
+ def add_options(self):
+ super(PostgreSQLCLI, self).add_options()
+ self.add_hostoption()
+ self.add_useroption()
+ self.add_opt('-d', '--database', help='Database to connect to')
+
+ def process_options(self):
+ super(PostgreSQLCLI, self).process_options()
+ self.host = self.get_opt('host')
+ self.host = self.get_opt('host')
+ self.user = self.get_opt('user')
+ self.password = self.get_opt('password')
+ validate_host(self.host)
+ validate_port(self.port)
+ validate_user(self.user)
+ validate_password(self.password)
+ self.port = int(self.port)
+ #self.ssl = self.get_opt('ssl')
+
+ def connect(self, database):
+ log.info('connecting to %s:%s database %s as user %s', self.host, self.port, database)
+ return psycopg2.connect(host=self.host,
+ port=self.port,
+ database=database,
+ user=self.user,
+ password=self.password)
diff --git a/opentsdb_import_metric_distribution.py b/opentsdb_import_metric_distribution.py
index 1331b68d2..3b3d3df09 100755
--- a/opentsdb_import_metric_distribution.py
+++ b/opentsdb_import_metric_distribution.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-10 11:47:12 +0100 (Mon, 10 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/opentsdb_list_metrics.sh b/opentsdb_list_metrics.sh
index 3a2675738..a119bd9fa 100755
--- a/opentsdb_list_metrics.sh
+++ b/opentsdb_list_metrics.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-13 22:36:14 +0100 (Fri, 13 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -72,7 +72,7 @@ done
check_bin(){
local bin="$1"
- if ! type -P $bin &>/dev/null; then
+ if ! type -P "$bin" &>/dev/null; then
echo "'$bin' command not found in \$PATH ($PATH)"
exit 1
fi
@@ -83,6 +83,8 @@ check_bin jq
if [ -z "${DEBUG:-}" ]; then
curl_options="$curl_options -s"
fi
+# split opts
+# shellcheck disable=SC2086
curl $curl_options "$tsd_url/api/suggest?type=$metrics&q=&max=2000000000" |
jq '.[]' |
sed 's/"//g' |
diff --git a/opentsdb_list_metrics_hbase.sh b/opentsdb_list_metrics_hbase.sh
index 56aaddfa9..86142e4c0 100755
--- a/opentsdb_list_metrics_hbase.sh
+++ b/opentsdb_list_metrics_hbase.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-13 22:36:14 +0100 (Fri, 13 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -73,7 +73,7 @@ done
check_bin(){
local bin="$1"
- if ! type -P $bin &>/dev/null; then
+ if ! type -P "$bin" &>/dev/null; then
echo "'$bin' command not found in \$PATH ($PATH)"
exit 1
fi
@@ -91,6 +91,7 @@ print_metrics(){
print_metrics_by_age(){
# hackish but convenient - forking to the date command thousands or hundreds of thousands of times can take hours, python takes 10 secs even for 250,000+ metrics
tmp_python_script=$(mktemp)
+ # shellcheck disable=SC1117
cat > "$tmp_python_script" < 'name:$metrics', VERSIONS => 1 }" 2>/dev/null |
diff --git a/pig_udfs.jy b/pig_udfs.jy
index 845a1f531..571e2cedc 100755
--- a/pig_udfs.jy
+++ b/pig_udfs.jy
@@ -3,7 +3,7 @@
# Author: Hari Sekhon
# Date: 2015-03-12 21:16:54 +0000 (Thu, 12 Mar 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/plot_uk_marriage_rates.py b/plot_uk_marriage_rates.py
new file mode 100755
index 000000000..a38332443
--- /dev/null
+++ b/plot_uk_marriage_rates.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2025-04-26 00:37:28 +0800 (Sat, 26 Apr 2025)
+#
+# https///github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Plots the Marriage rates in England & Wales using the latest download from the UK Government website:
+
+ https://www.ons.gov.uk/peoplepopulationandcommunity/birthsdeathsandmarriages/marriagecohabitationandcivilpartnerships/bulletins/marriagesinenglandandwalesprovisional/2021and2022
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import platform
+import subprocess
+import sys
+#import time
+import traceback
+import pandas as pd
+import matplotlib.pyplot as plt
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log
+ from harisekhon import CLI
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1'
+
+
+# pylint: disable=too-few-public-methods
+class PlotUKMarriageRates(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(PlotUKMarriageRates, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.timeout_default = 0
+
+ # def add_options(self):
+ # super(PlotUKMarriageRates, self).add_options()
+ #
+ # def process_options(self):
+ # super(PlotUKMarriageRates, self).process_options()
+
+ def run(self):
+ file_path = f"{srcdir}/tests/data/uk_marriage_rates_2022.xslx"
+ if self.args:
+ file_path = self.args[0]
+ #self.usage("Provide path to datadownload.xlsx")
+
+ if not os.path.isfile(file_path):
+ self.usage(f"Invalid argument provided, not a file: {file_path}")
+
+ log.info(f"Loading file: {file_path}")
+ xls = pd.ExcelFile(file_path)
+
+ log.info("Parsing xls")
+ # Read the relevant sheet and skip the metadata
+ df = xls.parse('Figure 2', skiprows=6)
+
+ # Set proper column headers
+ df.columns = df.iloc[1]
+ df = df[2:] # Drop the header rows
+
+ # Rename columns for clarity
+ df.columns = ['Year', 'Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women']
+
+ # Convert data types
+ df['Year'] = df['Year'].astype(int)
+ for col in ['Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women']:
+ df[col] = pd.to_numeric(df[col], errors='coerce')
+
+ log.info("Plotting")
+ plt.figure(figsize=(10, 6))
+ plt.plot(df['Year'], df['Opposite-sex Men'], label='Opposite-sex Men')
+ plt.plot(df['Year'], df['Opposite-sex Women'], label='Opposite-sex Women')
+ plt.plot(df['Year'], df['Same-sex Men'], label='Same-sex Men')
+ plt.plot(df['Year'], df['Same-sex Women'], label='Same-sex Women')
+
+ plt.title('Marriage Rates Over Time (England & Wales)')
+ plt.xlabel('Year')
+ plt.ylabel('Marriage Rate per 1,000 People')
+ plt.legend()
+ plt.grid(True)
+ plt.tight_layout()
+ #plt.show()
+ #
+ # doesn't show anything because the script finishes before the GUI event loop
+ # has time to process and display the window
+ #plt.show(block=False)
+ #
+ # pylint: disable=line-too-long
+ #
+ # doesn't work even with this hack:
+ #
+ # WARNING: NSWindow geometry should only be modified on the main thread! This will raise an exception in the future
+ #
+ #import threading
+ #threading.Thread(target=plt.show).start()
+ #sleep_secs = 20
+ #log.info(f"Sleeping for {sleep_secs} secs to allow you to see the graph pop-up")
+ #time.sleep(sleep_secs)
+
+ image_path = os.path.splitext(file_path)[0] + '.png'
+ log.info("Generating output image: {image_path}")
+
+ #if os.path.exists(image_path):
+ # log.warning(f"Image page already exists, skipping recreating for safety: {image_path}")
+ # #plt.close()
+ # return
+
+ # doesn't solve blank png
+ #plt.gcf().canvas.draw()
+
+ # results in blank png
+ #plt.savefig(image_path)
+
+ fig = plt.gcf() # draw before saving
+ fig.canvas.draw() # force rendering
+ plt.savefig(image_path)
+
+ plt.show()
+
+ if platform.system() == "Darwin":
+ # fire and forget
+ # pylint: disable=subprocess-run-check
+ subprocess.run(['open', image_path])
+
+ #plt.close()
+
+
+if __name__ == '__main__':
+ PlotUKMarriageRates().main()
diff --git a/pylib b/pylib
index 93cc037e0..0d272f4cf 160000
--- a/pylib
+++ b/pylib
@@ -1 +1 @@
-Subproject commit 93cc037e06a8f4c360369e0e76a0550dc2822976
+Subproject commit 0d272f4cf2c27a8ee614eea18c0bd171e47a06cd
diff --git a/python_find_library_path.py b/python_find_library_path.py
index 24d0aab2a..66fb9c00e 100755
--- a/python_find_library_path.py
+++ b/python_find_library_path.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-09-27
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/pythonpath.py b/pythonpath.py
index 2d9d8f4d6..ef42891b9 100755
--- a/pythonpath.py
+++ b/pythonpath.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-09-27
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/quay_show_tags.py b/quay_show_tags.py
index 5743be955..d1eef86bf 100755
--- a/quay_show_tags.py
+++ b/quay_show_tags.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-10 11:26:49 +0100 (Tue, 10 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -30,6 +30,7 @@
#from __future__ import unicode_literals
import os
+import re
import sys
import traceback
srcdir = os.path.abspath(os.path.dirname(__file__))
@@ -43,7 +44,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.6.2'
+__version__ = '0.6.3'
class QuayTags(DockerHubTags):
@@ -61,7 +62,10 @@ def run(self):
self.quiet = self.get_opt('quiet')
if not self.quiet:
print('\nQuay.io ', end='')
+ re_quay = re.compile('^quay.io/', re.I)
for arg in self.args:
+ if re_quay.match(arg):
+ arg = re_quay.sub('', arg)
self.print_tags(arg)
diff --git a/requirements.txt b/requirements.txt
index bd5bdaa1c..05d204c50 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,50 +1,97 @@
avro==1.8.1
+
# requires Python == 3.4, build in Makefile instead
#avro-python3==1.9.0
-awscli==1.16.241
+
+# AWS CLIv1 is obsolete and doesn't support SSO - use CLIv2 - see https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/install/install_aws_cli.sh
+#awscli==1.16.241
#bitarray==0.8.1
+
+#boto==2.49.0
+boto3==1.10.37
#cassandra-driver==3.6.0
dicttoxml==1.7.4
+
# Elasticsearch library must match major version :(
# for Elasticsearch 2.x
#elasticsearch>=2.0.0,<3.0.0
# for Elasticsearch 1.x
#elasticsearch>=1.0.0,<2.0.0
+
# fails on requiring newer version of setuptools
#Flask==0.10.1
-GitPython==2.1.14
-happybase==1.0.0
+GitPython==2.1.15
+
+# this GCP API is surprisingly awful, not using
+#google-api-python-client==1.11.0
+
+# XXX: install broken on new M1 Mac Python 3.9
+#happybase==1.0.0
+
humanize==0.5.1
-impyla==0.16.0
-Jinja2==2.10.1
+impyla==0.19.0
+jinja2==2.11.3
#kazoo==2.2.1
ldif3==3.2.2
#MarkupSafe==0.23
#Markdown==2.6.8
+
+matplotlib==3.7.5
+
# Python 3.5+
#numpy==1.17.2
-numpy==1.16.5
+# XXX: install broken on new M1 Mac Python 3.9
+#numpy==1.16.5
+
+# for plot_uk_marriage_rates.py
+openpyxl==3.1.5
+pandas==2.0.3
+
+# requires pg_config to build from source
+#psycopg2==2.8.4
+# XXX: install broken on new M1 Mac Python 3.9
+#psycopg2-binary==2.8.4
+
python-cson==1.0.9
-psutil==4.3.0
+psutil==5.7.0
+
# parquet support in pyarrow is weaker, gone back to using parquet-tools
#pyarrow==0.6.0
#PyHive==0.6.1
+
+# doesn't work with non-trivial code
#PyInstaller==3.3.1
-python-ldap==3.2.0
+
+# gcc compile error on Alpine, don't think this is used either
+#python-ldap==3.2.0
+
#python-jenkins==0.4.13
# pulled in automatically by snakebite[kerberos]
#python-krbV==1.0.90
# needed by avro
-python-snappy==0.5
-sasl==0.2.1
+
+# XXX: install broken on new M1 Mac Python 3.9
+#python-snappy==0.5
+
+# XXX: install broken on new M1 Mac Python 3.9
+#sasl==0.2.1
+
sh==1.12.14
+selenium==3.141.0
+
# pulls in python-KrbV as a dependency which doesn't build on Mac any more
-# moved to Makefile as best effort
+# relies on python-krbV is unmaintained and unported to Python 3
+# - moved to Makefile as best effort
#snakebite[kerberos]==2.11.0
-snakebite==2.11.0
-thrift-sasl==0.2.1
-thrift==0.9.3
-thriftpy==0.3.9
+#snakebite==2.11.0
+
+# XXX: install broken on new M1 Mac Python 3.9
+#thrift-sasl==0.2.1
+#thrift==0.9.3
+#thriftpy==0.3.9
+
toml==0.10.0
xmltodict==0.10.2
yamllint==1.15.0
+
+#pyyaml>=5.4 # not directly required, pinned by Snyk to avoid a vulnerability. update: this breaks Python 3.5 build where this requirement is not found
diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py
new file mode 100755
index 000000000..c3d117e53
--- /dev/null
+++ b/selenium_hub_browser_test.py
@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2021-05-12 09:55:01 +0100 (Wed, 12 May 2021)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tests a Selenium Hub / Selenoid using the given browsers eg. FIREFOX, CHROME
+against a given URL and content (defaults to google.com)
+
+Browsers default to 'FIREFOX' and 'CHROME' if not specified
+URL defaults to 'google.com' checking for content 'google'
+If you define a different URL then you must specify a --content or --regex validation otherwise none is used
+
+Example:
+
+ ./selenium_hub_browser_test.py --host [] []
+
+ ./selenium_hub_browser_test.py --hub-url http://:4444/wd/hub/ [] []
+
+Where browsers are one or more of these and must be supported by the remote Selenium Hub:
+
+ANDROID
+CHROME
+EDGE
+FIREFOX
+HTMLUNIT
+HTMLUNITWITHJS
+INTERNETEXPLORER
+IPAD
+IPHONE
+OPERA
+PHANTOMJS
+SAFARI
+WEBKITGTK
+
+Examples:
+
+ ./selenium_hub_browser_test.py --host x.x.x.x
+
+ ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME
+
+ ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --content google
+ ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --regex 'goog.*'
+
+
+Tested on Selenium Grid Hub v.3.141.59, v4.0.0 and Selenoid 1.10.1
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import re
+import sys
+import time
+import traceback
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon import CLI
+ from harisekhon.utils import log, validate_host, validate_port, validate_url, validate_regex, die
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.3'
+
+
+class SeleniumHubBrowserTest(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(SeleniumHubBrowserTest, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.host = None
+ self.port = None
+ self.protocol = 'http'
+ self.name = 'Selenium Hub'
+ self.path = 'wd/hub'
+ self.hub_url = None
+ self.url_default = 'http://google.com'
+ self.url = self.url_default
+ self.expected_content = None
+ self.expected_content_default = 'google'
+ self.expected_regex = None
+ self.timeout_default = 600
+ self.verbose_default = 2
+
+ def add_options(self):
+ super(SeleniumHubBrowserTest, self).add_options()
+ self.add_hostoption(name='Selenium Hub', default_port=4444)
+ self.add_opt('-U', '--hub-url', help='Selenium Hub URL (takes priority over --host/--port/--ssl)')
+ self.add_opt('-u', '--url', default=self.url_default,
+ help='URL to use for the test (default: {})'.format(self.url_default))
+ self.add_opt('-c', '--content', help='URL content to expect')
+ self.add_opt('-r', '--regex', help='URL content to expect')
+ self.add_opt('-S', '--ssl', action='store_true', help='Use SSL to connect to Selenium Hub')
+
+ def process_options(self):
+ super(SeleniumHubBrowserTest, self).process_options()
+ self.hub_url = self.get_opt('hub_url')
+ if self.hub_url:
+ validate_url(self.hub_url, 'hub')
+ else:
+ self.host = self.get_opt('host')
+ self.port = self.get_opt('port')
+ validate_host(self.host)
+ validate_port(self.port)
+ if self.get_opt('ssl') or int(self.port) == 443:
+ self.protocol = 'https'
+ self.hub_url = '{protocol}://{host}:{port}/{path}'\
+ .format(protocol=self.protocol, \
+ host=self.host, \
+ port=self.port, \
+ path=self.path)
+ self.url = self.get_opt('url')
+ if ':' not in self.url:
+ self.url = 'http://' + self.url
+ validate_url(self.url)
+ self.expected_content = self.get_opt('content')
+ self.expected_regex = self.get_opt('regex')
+ if self.expected_regex:
+ validate_regex(self.expected_regex)
+ self.expected_regex = re.compile(self.expected_regex)
+ elif self.url == self.url_default:
+ self.expected_content = self.expected_content_default
+ if not self.args:
+ # test basic Chrome and Firefox are available
+ self.args.append('chrome')
+ self.args.append('firefox')
+
+ def check_selenium(self, browser):
+ log.info("Connecting to '%s' for browser '%s'", self.hub_url, browser)
+ driver = webdriver.Remote(
+ command_executor=self.hub_url,
+ desired_capabilities=getattr(DesiredCapabilities, browser)
+ )
+ log.info("Checking url '%s'", self.url)
+ driver.get(self.url)
+ content = driver.page_source
+ title = driver.title
+ driver.quit()
+ if self.expected_regex:
+ log.info("Checking url content matches regex")
+ if not self.expected_regex.search(content):
+ die('ERROR: Page source content failed regex search')
+ elif self.expected_content:
+ log.info("Checking url content matches '%s'", self.expected_content)
+ if self.expected_content not in content:
+ die('ERROR: Page source content failed content match')
+ # not really recommended but in this case we cannot predict
+ # what to expect on a random url if not specified by --content/--regex (provided in the default test case)
+ #
+ # https://www.selenium.dev/documentation/en/worst_practices/http_response_codes/
+ elif '404' in title:
+ die('ERROR: Page title contains a 404 / error ' +
+ '(if this is expected, specify --content / --regex to check instead): {}'.format(title))
+ log.info("Succeeded for browser '%s' against url '%s'", browser, self.url)
+
+ def run(self):
+ start_time = time.time()
+ for browser in self.args:
+ self.check_selenium(browser.upper())
+ query_time = time.time() - start_time
+ log.info('Finished checks in {:.2f} secs'.format(query_time))
+
+
+if __name__ == '__main__':
+ SeleniumHubBrowserTest().main()
diff --git a/serf_event_handler.py b/serf_event_handler.py
index 38a0df1b4..c0d5cacbb 100755
--- a/serf_event_handler.py
+++ b/serf_event_handler.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-01-16 15:44:16 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -61,7 +61,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.2.2'
+__version__ = '0.2.3'
class SerfEventHandler(CLI):
@@ -95,7 +95,7 @@ def add_options(self):
def enable_commands(self):
if self.event in ['query', 'event']:
- cmd = None
+ cmd = ''
if self.event == 'query':
cmd = self.query_name
elif self.event == 'user':
diff --git a/setup/apk-packages-dev.txt b/setup/apk-packages-dev.txt
index f8738775b..a8fe03ea8 100644
--- a/setup/apk-packages-dev.txt
+++ b/setup/apk-packages-dev.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
@@ -14,5 +14,8 @@
# ============================================================================ #
openldap-dev
+postgresql-dev
snappy-dev
-unzip
+
+# installed by bash-tools submodule now
+#unzip
diff --git a/setup/apk-packages-pip.txt b/setup/apk-packages-pip.txt
index 88bcf4c3a..1b9ee9d77 100644
--- a/setup/apk-packages-pip.txt
+++ b/setup/apk-packages-pip.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
@@ -16,6 +16,8 @@
py-dicttoxml
py-psutil
py-pyldap
-py2-jinja2
-py2-numpy
-#py-flask
+py3-jinja2
+py3-numpy
+#py3-flask
+py3-pygit2
+py3-psycopg2
diff --git a/setup/apk-packages.txt b/setup/apk-packages.txt
index fc6956249..267f430e5 100644
--- a/setup/apk-packages.txt
+++ b/setup/apk-packages.txt
@@ -2,35 +2,38 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
# Alpine Package Requirements
# ============================================================================ #
-# full mktemp needed for json tests
-# full split needed for anonymize_parallel.sh
-coreutils
-
-grep
-
-# needed for tests/test_spark* and ambari_cancel_all_requests.sh
-curl
-
# for htpasswd for docker registry authenticated checks
apache2-utils
-# for anonymize_parallel.sh
-parallel
-
#which java || $(SUDO) apk add openjdk8-jre-base
# Spark Java Py4J gets java linking error without this
#if [ -f /lib/libc.musl-x86_64.so.1 ]; then [ -e /lib/ld-linux-x86-64.so.2 ] || ln -sv /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2; fi
-zip
+# =====================================
+# installed by bash-tools submodule now
+
+# full mktemp needed for json tests
+# full split needed for anonymize_parallel.sh
+#coreutils
+
+# needed for tests/test_spark* and ambari_cancel_all_requests.sh
+#curl
+
+#grep
+
+# for anonymize_parallel.sh
+#parallel
+
+#zip
diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh
new file mode 100755
index 000000000..d7bedf92a
--- /dev/null
+++ b/setup/bootstrap.sh
@@ -0,0 +1,86 @@
+#!/bin/sh
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-10-16 10:33:03 +0100 (Wed, 16 Oct 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# Alpine / Wget:
+#
+# wget -O- https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh | sh
+#
+# Curl:
+#
+# curl https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh | sh
+
+set -eu
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(dirname "$0")"
+
+repo="https://github.com/HariSekhon/DevOps-Python-tools"
+
+directory="pytools"
+
+sudo=""
+[ "$(whoami)" = "root" ] || sudo=sudo
+
+if [ "$(uname -s)" = Darwin ]; then
+ echo "Bootstrapping on Mac OS X: $repo"
+ if ! type brew >/dev/null 2>&1; then
+ curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install | $sudo ruby
+ fi
+elif [ "$(uname -s)" = Linux ]; then
+ echo "Bootstrapping on Linux: $repo"
+ if type apk >/dev/null 2>&1; then
+ $sudo apk --no-cache add bash git make curl wget
+ elif type apt-get >/dev/null 2>&1; then
+ if [ -n "${CI:-}" ]; then
+ export DEBIAN_FRONTEND=noninteractive
+ fi
+ opts="-o DPkg::Lock::Timeout=1200"
+ if [ -z "${PS1:-}" ]; then
+ opts="$opts -qq"
+ fi
+ $sudo apt-get update $opts
+ $sudo apt-get install $opts -y git make curl wget --no-install-recommends
+ elif type yum >/dev/null 2>&1; then
+ if grep -qi 'NAME=.*CentOS' /etc/*release; then
+ echo "CentOS EOL detected, replacing yum base URL to vault to re-enable package installs"
+ $sudo sed -i 's/^[[:space:]]*mirrorlist/#mirrorlist/' /etc/yum.repos.d/CentOS-Linux-*
+ $sudo sed -i 's|^#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|' /etc/yum.repos.d/CentOS-Linux-*
+ fi
+ $sudo yum install -y git make curl wget
+ else
+ echo "Package Manager not found on Linux, cannot bootstrap"
+ exit 1
+ fi
+else
+ echo "Only Mac & Linux are supported for conveniently bootstrapping all install scripts at this time"
+ exit 1
+fi
+
+if [ "${srcdir##*/}" = setup ]; then
+ cd "$srcdir/.."
+else
+ # if this is an empty directory eg. a cache mount, then remove it to get a proper checkout
+ rmdir "$directory" 2>/dev/null || :
+ if [ -d "$directory" ]; then
+ cd "$directory"
+ git pull
+ else
+ git clone "$repo" "$directory"
+ cd "$directory"
+ fi
+fi
+
+if [ -z "${NO_MAKE:-}" ]; then
+ make
+fi
diff --git a/setup/brew-packages.txt b/setup/brew-packages.txt
index fa25317ba..401e24b97 100644
--- a/setup/brew-packages.txt
+++ b/setup/brew-packages.txt
@@ -3,19 +3,20 @@
# Author: Hari Sekhon
# Date: 2018-05-22 11:15:16 +0100 (Tue, 22 May 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
# Mac OS X - Homebrew Package Requirements
# ============================================================================ #
+# installed by bash-tools submodule now
# for anonymize_parallel.sh
-parallel
+#parallel
parquet-tools
snappy
diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh
new file mode 100755
index 000000000..b01a77d04
--- /dev/null
+++ b/setup/ci_bootstrap.sh
@@ -0,0 +1,94 @@
+#!/bin/sh
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-06-02 17:43:35 +0100 (Tue, 02 Jun 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# Designed to bootstrap all CI systems with retries to make sure the networking, package lists and package repos works before proceeding
+#
+# Minimizes CI build failures due to temporary networking blips, which happens more often than you would think when you have a large number of CI builds across a lot of disparate systems
+
+set -eu
+[ -n "${DEBUG:-}" ] && set -x
+
+max_tries=10
+interval=60 # secs
+
+sudo=""
+# EUID undefined in posix sh
+#[ $EUID = 0 ] || sudo=sudo
+[ "$(whoami)" = root ] || sudo=sudo
+
+retry(){
+ # no local in posix sh
+ count=0
+ while true; do
+ # no let or bare (()) in posix sh, must discard output rather than execute it
+ _=$((count+=1))
+ printf "%s try %d: " "$(date '+%F %T')" "$count"
+ echo "$*"
+ "$@" &&
+ break;
+ echo
+ if [ "$count" -ge "$max_tries" ]; then
+ echo "$count tries failed, aborting..."
+ exit 1
+ fi
+ echo "sleeping for $interval secs before retrying"
+ sleep "$interval"
+ echo
+ done
+}
+
+if [ "$(uname -s)" = Darwin ]; then
+ echo "Bootstrapping Mac"
+ # removing adjacent dependency to be able to curl from github to avoid submodule circular dependency (git / submodule / install git & make)
+ #retry "$srcdir/../install/install_homebrew.sh"
+ if command -v brew 2>&1; then
+ # fix for CI runners on Mac with shallow homebrew clone - which is failing all the BuildKite builds
+ for git_root in /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask; do
+ if [ -d "$git_root" ]; then
+ # find out if Homebrew is a shallow git checkout and if so fix it
+ if [ -f "$(git -C "$git_root" rev-parse --git-dir)/shallow" ] ||
+ [ "$(git -C "$git_root" rev-parse --is-shallow-repository)" = true ]; then
+ git -C "$git_root" fetch --unshallow
+ fi
+ fi
+ done
+ retry brew update
+ fi
+elif [ "$(uname -s)" = Linux ]; then
+ echo "Bootstrapping Linux"
+ if type apk >/dev/null 2>&1; then
+ retry $sudo apk update
+ retry $sudo apk add --no-progress bash git make
+ elif type apt-get >/dev/null 2>&1; then
+ opts="-q -o DPkg::Lock::Timeout=1200"
+ retry $sudo apt-get update $opts
+ retry $sudo apt-get install $opts -y git make
+ elif type yum >/dev/null 2>&1; then
+ #retry $sudo yum makecache
+ retry $sudo yum install -qy git make
+ else
+ echo "Package Manager not found on Linux, cannot bootstrap"
+ exit 1
+ fi
+else
+ echo "Only Mac & Linux are supported for conveniently bootstrapping all install scripts at this time"
+ exit 1
+fi
+
+#retry make init
+
+# not calling make because in some CI systems we call 'make ci' which includes retries but in others with more restrictive build minutes we only run 'make' for a single shot build
+#
+#make
diff --git a/setup/ci_git_set_dir_safe.sh b/setup/ci_git_set_dir_safe.sh
new file mode 100755
index 000000000..66a53903a
--- /dev/null
+++ b/setup/ci_git_set_dir_safe.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2022-08-03 20:07:09 +0100 (Wed, 03 Aug 2022)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# Necessary for some CI/CD systems like Azure DevOps Pipelines which have incorrect ownership on the git checkout dir triggering this error:
+#
+# fatal: detected dubious ownership in repository at '/code/sql'
+
+# standalone script without lib dependency so it can be called directly from bootstrapped CI before submodules, since that is the exact problem that needs to be solved to allow CI/CD systems with incorrect ownership of the checkout directory to be able to checkout the necessary git submodules
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+dir="${1:-$srcdir/..}"
+
+cd "$dir"
+
+echo "Setting directory as safe: $PWD"
+git config --global --add safe.directory "$PWD"
+
+while read -r submodule_dir; do
+ dir="$PWD/$submodule_dir"
+ echo "Setting directory as safe: $dir"
+ git config --global --add safe.directory "$dir"
+done < <(git submodule | awk '{print $2}')
+
+echo "Done"
diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt
index a2e311a6b..015439134 100644
--- a/setup/deb-packages-dev.txt
+++ b/setup/deb-packages-dev.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
@@ -14,6 +14,7 @@
# ============================================================================ #
libldap2-dev
+libpq-dev # postgres pg_config
# needed to build python-snappy for avro module
libsnappy-dev
diff --git a/setup/deb-packages-optional.txt b/setup/deb-packages-optional.txt
index 038ae8802..cd797f747 100644
--- a/setup/deb-packages-optional.txt
+++ b/setup/deb-packages-optional.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
diff --git a/setup/deb-packages-pip.txt b/setup/deb-packages-pip.txt
index 75196cfb9..da4eea8c8 100644
--- a/setup/deb-packages-pip.txt
+++ b/setup/deb-packages-pip.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
@@ -24,6 +24,7 @@ python-ldap
python-ldif3
python-numpy
python-psutil
+python-psycopg2
python-sh
python-snappy
python-thrift
diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt
index c605fdd27..4a028e4b1 100644
--- a/setup/deb-packages.txt
+++ b/setup/deb-packages.txt
@@ -2,33 +2,20 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
# Deb Package Requirements
# ============================================================================ #
-# needed for tests/test_spark* and ambari_cancel_all_requests.sh
-curl
-
-# either of these should do to ensure ping command is present for find_active_server.py's ping mode
-iputils-ping
-#inetutils-ping
-
# for htpasswd for docker registry authenticated checks
apache2-utils
-# for anonymize_parallel.sh
-parallel
-
-# needed for serf test's 'uptime' command
-procps
-
# installs 343MB of dependencies - install by hand if needed
#ffmpeg
@@ -41,5 +28,21 @@ procps
#which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk
-zip
-unzip
+# =====================================
+# installed by bash-tools submodule now
+
+# needed for tests/test_spark* and ambari_cancel_all_requests.sh
+#curl
+
+# either of these should do to ensure ping command is present for find_active_server.py's ping mode
+#iputils-ping
+##inetutils-ping
+
+# for anonymize_parallel.sh
+#parallel
+
+# needed for serf test's 'uptime' command
+#procps
+
+#zip
+#unzip
diff --git a/setup/gocd_config_repo.json b/setup/gocd_config_repo.json
new file mode 100644
index 000000000..ce63bf63d
--- /dev/null
+++ b/setup/gocd_config_repo.json
@@ -0,0 +1,26 @@
+{
+ "id": "DevOps-Python-tools",
+ "plugin_id": "yaml.config.plugin",
+ "material": {
+ "type": "git",
+ "attributes": {
+ "url": "https://github.com/HariSekhon/DevOps-Python-tools",
+ "branch": "master",
+ "auto_update": true
+ }
+ },
+ "configuration": [
+ {
+ "key": "file_pattern",
+ "value": "cicd/*.gocd.y*ml"
+ }
+ ],
+ "rules": [
+ {
+ "directive": "allow",
+ "action": "*",
+ "type": "*",
+ "resource": "*"
+ }
+ ]
+}
diff --git a/setup/jenkins-job.xml b/setup/jenkins-job.xml
new file mode 100644
index 000000000..2c9da2b6b
--- /dev/null
+++ b/setup/jenkins-job.xml
@@ -0,0 +1,57 @@
+
+
+
+
+
+
+
+ hudson.triggers.SCMTrigger
+ hudson.triggers.TimerTrigger
+
+
+
+
+
+
+ false
+
+
+
+ https://github.com/HariSekhon/DevOps-Python-tools/
+
+
+
+
+
+ H 10 * * 1-5
+
+
+ H/2 * * * *
+ false
+
+
+
+
+
+
+ 2
+
+
+ https://github.com/HariSekhon/DevOps-Python-tools
+
+
+
+
+ */master
+
+
+ false
+
+
+
+ Jenkinsfile
+ true
+
+
+ false
+
\ No newline at end of file
diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt
index a0015d809..f36770476 100644
--- a/setup/rpm-packages-dev.txt
+++ b/setup/rpm-packages-dev.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
@@ -17,6 +17,9 @@ gcc-c++
# needed to build python-krbV and cloudera/thrift_sasl
cyrus-sasl-devel
krb5-devel
+openldap-devel
openssl-devel
+
+# moved to optional to account for changed package names on CentOS 8
# needed to build python-snappy for avro module
-snappy-devel
+#snappy-devel
diff --git a/setup/rpm-packages-optional.txt b/setup/rpm-packages-optional.txt
index f94b6019d..b1bb97d9e 100644
--- a/setup/rpm-packages-optional.txt
+++ b/setup/rpm-packages-optional.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
@@ -14,3 +14,14 @@
# ============================================================================ #
yamllint
+
+# CentOS <= 7
+snappy-devel
+# CentOS 8
+csnappy-devel
+
+# postgres pg_config
+# CentOS 8
+libpq-devel
+# CentOS 7
+postgresql-devel
diff --git a/setup/rpm-packages-pip.txt b/setup/rpm-packages-pip.txt
index db112cbc7..2a33f4bc5 100644
--- a/setup/rpm-packages-pip.txt
+++ b/setup/rpm-packages-pip.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
@@ -25,3 +25,8 @@ python2-psutil
#python-flask
#python-markupsafe
#python2-bitarray
+
+python2-psycopg2
+python3-psycopg2
+
+python3-snappy
diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt
index 281cfb022..7f6263457 100644
--- a/setup/rpm-packages.txt
+++ b/setup/rpm-packages.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
@@ -15,21 +15,24 @@
java
-# needed for tests/test_spark* and ambari_cancel_all_requests.sh
-curl
-
-# for ping mode of find_active_server.py
-iputils
-
# for htpasswd for docker registry authenticated checks
-https-tools
+httpd-tools
# needed to build pyhs2
# libgsasl-devel saslwrapper-devel
#cyrus-sasl-devel
+# =====================================
+# installed by bash-tools submodule now
+
+# needed for tests/test_spark* and ambari_cancel_all_requests.sh
+#curl
+
+# for ping mode of find_active_server.py
+#iputils
+
# for anonymize_parallel.sh
-parallel
+#parallel
-unzip
-zip
+#unzip
+#zip
diff --git a/sonar-project.properties b/sonar-project.properties
index 75e1bf83c..319cc6897 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -1,32 +1,42 @@
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
-# Date: 2016-07-19 18:18:27 +0100 (Tue, 19 Jul 2016)
+# Date: 2016-07-19 18:31:17 +0100 (Tue, 19 Jul 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
-sonar.projectName=DevOps Python Tools
-sonar.projectKey=pytools
+# ============================================================================ #
+# S o n a r Q u b e
+# ============================================================================ #
+
+sonar.host.url=https://sonarcloud.io
+
+# Required metadata
+sonar.organization=harisekhon
+sonar.projectName=DevOps-Python-tools
+sonar.projectKey=HariSekhon_DevOps-Python-tools
sonar.projectVersion=1.0
-sonar.projectDescription=Python / Jython Tools
+sonar.projectDescription=DevOps-Python-tools
-sonar.links.homepage=https://github.com/harisekhon/devops-python-tools
-sonar.links.scm=https://github.com/harisekhon/devops-python-tools
-sonar.links.issue=https://github.com/harisekhon/devops-python-tools/issues
-sonar.links.ci=https://travis-ci.org/HariSekhon/devops-python-tools
+sonar.links.homepage=https://github.com/HariSekhon/DevOps-Python-tools
+sonar.links.scm=https://github.com/HariSekhon/DevOps-Python-tools
+sonar.links.issue=https://github.com/HariSekhon/DevOps-Python-tools/issues
+sonar.links.ci=https://github.com/HariSekhon/DevOps-Python-tools/actions
+# directories to scan (defaults to sonar-project.properties dir otherwise)
sonar.sources=.
#sonar.language=py
sonar.sourceEncoding=UTF-8
-sonar.exclusions=**/tests/spark*/**/*
+#sonar.exclusions=**/tests/**
+sonar.exclusions=**/zookeeper-*/**/*
diff --git a/spark_avro_to_parquet.py b/spark_avro_to_parquet.py
index 10fd55e88..8528af1a6 100755
--- a/spark_avro_to_parquet.py
+++ b/spark_avro_to_parquet.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -52,6 +52,7 @@
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py
index 955b395e7..d41717b4f 100755
--- a/spark_csv_to_avro.py
+++ b/spark_csv_to_avro.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -57,14 +57,15 @@
'com.databricks:spark-avro_2.10:2.0.1 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
-from pyspark.sql.types import * # pylint: disable=wrong-import-position,import-error,wildcard-import
+from pyspark.sql.types import * # lgtm [py/polluting-import] pylint: disable=wrong-import-position,import-error,wildcard-import
from pyspark.sql.types import StructType, StructField # pylint: disable=wrong-import-position,import-error
__author__ = 'Hari Sekhon'
-__version__ = '0.8.0'
+__version__ = '0.8.1'
class SparkCSVToAvro(CLI):
@@ -166,7 +167,6 @@ def create_struct(arg):
die("Spark version couldn't be determined. " + support_msg('pytools'))
# pylint: disable=invalid-name
- df = None
if isMinVersion(spark_version, 1.4):
if has_header and not schema:
log.info('inferring schema from CSV headers')
@@ -198,5 +198,6 @@ def create_struct(arg):
# the databricks avro driver
df.write.format('com.databricks.spark.avro').save(avro_dir)
+
if __name__ == '__main__':
SparkCSVToAvro().main()
diff --git a/spark_csv_to_parquet.py b/spark_csv_to_parquet.py
index f3f6909e6..e70f014f6 100755
--- a/spark_csv_to_parquet.py
+++ b/spark_csv_to_parquet.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -49,14 +49,15 @@
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-csv_2.11:1.5.0 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
-from pyspark.sql.types import * # pylint: disable=wrong-import-position,import-error,wildcard-import
+from pyspark.sql.types import * # lgtm [py/polluting-import] pylint: disable=wrong-import-position,import-error,wildcard-import
from pyspark.sql.types import StructType, StructField # pylint: disable=wrong-import-position,import-error
__author__ = 'Hari Sekhon'
-__version__ = '0.8.0'
+__version__ = '0.8.1'
class SparkCSVToParquet(CLI):
@@ -157,9 +158,7 @@ def create_struct(arg):
if not isVersionLax(spark_version):
die("Spark version couldn't be determined. " + support_msg('pytools'))
- # pylint: disable=invalid-name
-
- df = None
+ # pylint: disable=invalid-name
if isMinVersion(spark_version, 1.4):
if has_header and not schema:
log.info('inferring schema from CSV headers')
diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py
index 40c52ac96..db9d069ed 100755
--- a/spark_json_to_avro.py
+++ b/spark_json_to_avro.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -52,6 +52,7 @@
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
@@ -109,7 +110,6 @@ def run(self):
die("Spark version couldn't be determined. " + support_msg('pytools'))
# pylint: disable=invalid-name
- df = None
if isMinVersion(spark_version, 1.4):
df = sqlContext.read.json(json_file)
else:
diff --git a/spark_json_to_parquet.py b/spark_json_to_parquet.py
index 41ab27e0d..6976ce153 100755
--- a/spark_json_to_parquet.py
+++ b/spark_json_to_parquet.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -43,6 +43,7 @@
print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
sys.exit(4)
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
diff --git a/spark_parquet_to_avro.py b/spark_parquet_to_avro.py
index d6e49f060..030816963 100755
--- a/spark_parquet_to_avro.py
+++ b/spark_parquet_to_avro.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -52,6 +52,7 @@
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
diff --git a/sql b/sql
new file mode 160000
index 000000000..30d80bbd3
--- /dev/null
+++ b/sql
@@ -0,0 +1 @@
+Subproject commit 30d80bbd33f0e6c26c230f26575cb671c41c3e7d
diff --git a/strip_ansi_escape_codes.py b/strip_ansi_escape_codes.py
index 6fc517dc6..24738194a 100755
--- a/strip_ansi_escape_codes.py
+++ b/strip_ansi_escape_codes.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2018-09-09 23:06:06 +0100 (Sun, 09 Sep 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish # pylint: disable=line-too-long
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/teamcity/.teamcity.vcs.json b/teamcity/.teamcity.vcs.json
new file mode 100644
index 000000000..be61f126b
--- /dev/null
+++ b/teamcity/.teamcity.vcs.json
@@ -0,0 +1,57 @@
+{
+ "id": "TeamCity",
+ "name": "TeamCity",
+ "vcsName": "jetbrains.git",
+ "href": "/httpAuth/app/rest/vcs-roots/id:TeamCity",
+ "project": {
+ "id": "_Root",
+ "name": "",
+ "description": "Contains all other projects",
+ "href": "/httpAuth/app/rest/projects/id:_Root",
+ "webUrl": "http://localhost:8111/project.html?projectId=_Root"
+ },
+ "properties": {
+ "count": 9,
+ "property": [
+ {
+ "name": "agentCleanFilesPolicy",
+ "value": "ALL_UNTRACKED"
+ },
+ {
+ "name": "agentCleanPolicy",
+ "value": "ON_BRANCH_CHANGE"
+ },
+ {
+ "name": "authMethod",
+ "value": "ANONYMOUS"
+ },
+ {
+ "name": "branch",
+ "value": "master"
+ },
+ {
+ "name": "ignoreKnownHosts",
+ "value": "true"
+ },
+ {
+ "name": "submoduleCheckout",
+ "value": "CHECKOUT"
+ },
+ {
+ "name": "url",
+ "value": "https://github.com/HariSekhon/TeamCity-CI"
+ },
+ {
+ "name": "useAlternates",
+ "value": "true"
+ },
+ {
+ "name": "usernameStyle",
+ "value": "USERID"
+ }
+ ]
+ },
+ "vcsRootInstances": {
+ "href": "/httpAuth/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)"
+ }
+}
diff --git a/teamcity/.teamcity.vcs.oauth.json b/teamcity/.teamcity.vcs.oauth.json
new file mode 100644
index 000000000..f6f41ed7b
--- /dev/null
+++ b/teamcity/.teamcity.vcs.oauth.json
@@ -0,0 +1,61 @@
+{
+ "id": "TeamCity",
+ "name": "TeamCity",
+ "vcsName": "jetbrains.git",
+ "href": "/httpAuth/app/rest/vcs-roots/id:TeamCity",
+ "project": {
+ "id": "_Root",
+ "name": "",
+ "description": "Contains all other projects",
+ "href": "/httpAuth/app/rest/projects/id:_Root",
+ "webUrl": "http://localhost:8111/project.html?projectId=_Root"
+ },
+ "properties": {
+ "count": 9,
+ "property": [
+ {
+ "name": "agentCleanFilesPolicy",
+ "value": "ALL_UNTRACKED"
+ },
+ {
+ "name": "agentCleanPolicy",
+ "value": "ON_BRANCH_CHANGE"
+ },
+ {
+ "name": "authMethod",
+ "value": "PASSWORD"
+ },
+ {
+ "name": "branch",
+ "value": "master"
+ },
+ {
+ "name": "ignoreKnownHosts",
+ "value": "true"
+ },
+ {
+ "name": "submoduleCheckout",
+ "value": "CHECKOUT"
+ },
+ {
+ "name": "url",
+ "value": "https://github.com/HariSekhon/TeamCity-CI"
+ },
+ {
+ "name": "useAlternates",
+ "value": "true"
+ },
+ {
+ "name": "username",
+ "value": "HariSekhon"
+ },
+ {
+ "name": "usernameStyle",
+ "value": "USERID"
+ }
+ ]
+ },
+ "vcsRootInstances": {
+ "href": "/httpAuth/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)"
+ }
+}
diff --git a/teamcity/.teamcity.vcs.ssh.json b/teamcity/.teamcity.vcs.ssh.json
new file mode 100644
index 000000000..1b780675a
--- /dev/null
+++ b/teamcity/.teamcity.vcs.ssh.json
@@ -0,0 +1,65 @@
+{
+ "id": "TeamCity",
+ "name": "TeamCity",
+ "vcsName": "jetbrains.git",
+ "href": "/app/rest/vcs-roots/id:TeamCity",
+ "project": {
+ "id": "_Root",
+ "name": "",
+ "description": "Contains all other projects",
+ "href": "/app/rest/projects/id:_Root",
+ "webUrl": "http://localhost:8111/project.html?projectId=_Root"
+ },
+ "properties": {
+ "count": 11,
+ "property": [
+ {
+ "name": "agentCleanFilesPolicy",
+ "value": "ALL_UNTRACKED"
+ },
+ {
+ "name": "agentCleanPolicy",
+ "value": "ON_BRANCH_CHANGE"
+ },
+ {
+ "name": "authMethod",
+ "value": "TEAMCITY_SSH_KEY"
+ },
+ {
+ "name": "branch",
+ "value": "refs/heads/master"
+ },
+ {
+ "name": "ignoreKnownHosts",
+ "value": "true"
+ },
+ {
+ "name": "submoduleCheckout",
+ "value": "CHECKOUT"
+ },
+ {
+ "name": "teamcitySshKey",
+ "value": "VCS SSH Key"
+ },
+ {
+ "name": "url",
+ "value": "github.com:HariSekhon/TeamCity-CI"
+ },
+ {
+ "name": "useAlternates",
+ "value": "true"
+ },
+ {
+ "name": "username",
+ "value": "git"
+ },
+ {
+ "name": "usernameStyle",
+ "value": "USERID"
+ }
+ ]
+ },
+ "vcsRootInstances": {
+ "href": "/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)"
+ }
+}
diff --git a/templates b/templates
new file mode 160000
index 000000000..f59532dd6
--- /dev/null
+++ b/templates
@@ -0,0 +1 @@
+Subproject commit f59532dd67a583be3522814a15024a561b12c5ec
diff --git a/tests/all.sh b/tests/all.sh
index fabd97170..dbac7b8fb 100755
--- a/tests/all.sh
+++ b/tests/all.sh
@@ -4,19 +4,20 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+# shellcheck disable=SC1090
. "$srcdir/utils.sh"
# imported by utils.sh above
@@ -28,9 +29,11 @@ section "Running PyTools ALL"
# runs against . by default
cd "$srcdir/..";
-bash-tools/check_all.sh
+# shellcheck disable=SC1090
+# has to be included so that isExcluded function is inherited
+. "$srcdir/../bash-tools/checks/check_all.sh"
-tests/test_yamllint.sh
+#tests/test_yamllint.sh
# do help afterwards for Spark to be downloaded, and then help will find and use downloaded spark for SPARK_HOME
exit 0
@@ -38,4 +41,4 @@ exit 0
# pyspark not found
tests/help.sh
-bash-tools/run_tests.sh
+bash-tools/checks/run_tests.sh
diff --git a/tests/check.sh b/tests/check.sh
index 4541f684b..b3dba3c12 100755
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:47:43 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -21,12 +21,12 @@ check(){
msg=$2
echo
hr2
- echo $msg
+ echo "$msg"
hr2
echo
- echo cmd: $cmd
+ echo "cmd: $cmd"
echo
- if eval $cmd; then
+ if eval "$cmd"; then
echo
echo "SUCCESS"
else
diff --git a/tests/compile.sh b/tests/compile.sh
index e235cfd70..2890b07a3 100755
--- a/tests/compile.sh
+++ b/tests/compile.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,17 +19,18 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
-. ./tests/utils.sh
+# shellcheck disable=SC1090
+. "$srcdir/utils.sh"
hr
echo "Compiling all Python files"
hr
echo
-for x in $(find . -iname '*.py' -o -iname '*.jy'); do
- isExcluded "$x" && continue
- echo "compiling $x"
- python -m py_compile $x
-done
+while read -r filename; do
+ isExcluded "$filename" && continue
+ echo "compiling $filename"
+ python -m py_compile "$filename"
+done < <(find . -iname '*.py' -o -iname '*.jy')
echo
echo
diff --git a/tests/data/add_ou.ldif b/tests/data/add_ou.ldif
index c175b4bba..c9bc193b2 100644
--- a/tests/data/add_ou.ldif
+++ b/tests/data/add_ou.ldif
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2017-08-03 17:44:12 +0200 (Thu, 03 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
dn: OU=Hadoop-Cluster-1,OU=Hadoop,DC=harisekhon,DC=co,DC=uk
diff --git a/tests/data/ldap_download.sh b/tests/data/ldap_download.sh
index 1eb3dcd6f..8c7f4a158 100755
--- a/tests/data/ldap_download.sh
+++ b/tests/data/ldap_download.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2019-04-11 18:57:53 +0100 (Thu, 11 Apr 2019)
#
-# https://github.com/harisekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/data/ldap_upload.sh b/tests/data/ldap_upload.sh
index 0ff49ca02..60cdeb5ab 100755
--- a/tests/data/ldap_upload.sh
+++ b/tests/data/ldap_upload.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2019-04-11 18:57:53 +0100 (Thu, 11 Apr 2019)
#
-# https://github.com/harisekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/data/uk_marriage_rates_2022.xslx b/tests/data/uk_marriage_rates_2022.xslx
new file mode 100644
index 000000000..54efc3321
Binary files /dev/null and b/tests/data/uk_marriage_rates_2022.xslx differ
diff --git a/tests/docker/apache-drill-docker-compose.yml b/tests/docker/apache-drill-docker-compose.yml
index 50c1de358..d5ede5b8b 100644
--- a/tests/docker/apache-drill-docker-compose.yml
+++ b/tests/docker/apache-drill-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 16:18:36 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/common.yml b/tests/docker/common.yml
index 4c4236d1c..0b2a2d133 100644
--- a/tests/docker/common.yml
+++ b/tests/docker/common.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 15:16:43 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/elasticsearch-docker-compose.yml b/tests/docker/elasticsearch-docker-compose.yml
index bd6d9c190..4838ee391 100644
--- a/tests/docker/elasticsearch-docker-compose.yml
+++ b/tests/docker/elasticsearch-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 18:41:13 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/elasticsearch-elastic.co-docker-compose.yml b/tests/docker/elasticsearch-elastic.co-docker-compose.yml
index 992448ba6..6b7095ab6 100644
--- a/tests/docker/elasticsearch-elastic.co-docker-compose.yml
+++ b/tests/docker/elasticsearch-elastic.co-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 18:41:13 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/hadoop-docker-compose.yml b/tests/docker/hadoop-docker-compose.yml
index bef1fe74b..157d45c63 100644
--- a/tests/docker/hadoop-docker-compose.yml
+++ b/tests/docker/hadoop-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 21:25:07 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/hbase-docker-compose.yml b/tests/docker/hbase-docker-compose.yml
index 04029f2bb..8ab70732b 100644
--- a/tests/docker/hbase-docker-compose.yml
+++ b/tests/docker/hbase-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 22:13:19 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/presto-dev-docker-compose.yml b/tests/docker/presto-dev-docker-compose.yml
index 835b126f1..aaf4872f4 100644
--- a/tests/docker/presto-dev-docker-compose.yml
+++ b/tests/docker/presto-dev-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2017-09-13 14:47:23 +0200 (Wed, 13 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.1'
diff --git a/tests/docker/presto-docker-compose.yml b/tests/docker/presto-docker-compose.yml
index bb09ee3ed..2b9cc99ed 100644
--- a/tests/docker/presto-docker-compose.yml
+++ b/tests/docker/presto-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2017-09-13 14:47:23 +0200 (Wed, 13 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.1'
diff --git a/tests/docker/registry-docker-compose.yml b/tests/docker/registry-docker-compose.yml
index 7a66c2e3f..5fae25ab1 100644
--- a/tests/docker/registry-docker-compose.yml
+++ b/tests/docker/registry-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2017-09-12 17:27:50 +0200 (Tue, 12 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/solrcloud-docker-compose.yml b/tests/docker/solrcloud-docker-compose.yml
index 7799417a7..9cf4885fa 100644
--- a/tests/docker/solrcloud-docker-compose.yml
+++ b/tests/docker/solrcloud-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-13 14:01:04 +0000 (Tue, 13 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/excluded.sh b/tests/excluded.sh
index 68f62c58f..7535d3be6 100755
--- a/tests/excluded.sh
+++ b/tests/excluded.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# intended only to be sourced by utils.sh
@@ -22,8 +22,11 @@ set -eu
isExcluded(){
local prog="$1"
+ # shellcheck disable=SC2049
[[ "$prog" =~ ^\* ]] && return 0
- [[ "$prog" =~ spark_.*.py ]] && return 0
+ [[ "$prog" =~ spark_.*\.py ]] && return 0
+ [[ "$prog" =~ \.jy ]] && return 0
+ [[ "$prog" =~ hdfs_find_replication_factor_1\.py ]] && return 0 # python-krbV doesn't build on Python 3
#[[ $prog =~ ipython-notebook ]] && return 0
# this external git check is expensive, skip it when in CI as using fresh git checkouts
is_CI && return 1
diff --git a/tests/help.sh b/tests/help.sh
index 851451152..09b48dfe0 100755
--- a/tests/help.sh
+++ b/tests/help.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -22,7 +22,8 @@ cd "$srcdir/..";
# shellcheck disable=SC1091
. ./tests/utils.sh
-for x in $(echo ./*.py 2>/dev/null); do
+# shellcheck disable=SC2068
+for x in ${@:-$(echo ./*.py 2>/dev/null)}; do
isExcluded "$x" && continue
set +e
echo "$x --help"
@@ -32,8 +33,13 @@ for x in $(echo ./*.py 2>/dev/null); do
echo; hr
if [ $status = 0 ]; then
[[ "$x" =~ ambari_blueprints.py$ ]] && continue
- [[ "$x" =~ (hive|impala)_schemas_csv.py$ ]] && continue
[[ "$x" =~ pythonpath.py$ ]] && continue
+ [[ "$x" =~ aws_s3_presign.py$ ]] && continue
+ elif [ $status = 1 ]; then
+ if [[ "$x" =~ hdfs_find_replication_factor_1.py$ ]] &&
+ ! python -c 'import krbV'; then # best effort, not available on Mac any more
+ continue
+ fi
fi
[ $status = 3 ] || { echo "status code for $x --help was $status not expected 3"; exit 1; }
done
diff --git a/tests/python3.sh b/tests/python3.sh
index eeaf5f9f3..b231f8df4 100755
--- a/tests/python3.sh
+++ b/tests/python3.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
pip install caniusepython3
diff --git a/tests/syntax.sh b/tests/syntax.sh
index 26a8f16c8..5299ce323 100755
--- a/tests/syntax.sh
+++ b/tests/syntax.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,20 +19,21 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
-for x in $(echo *.py *.jy 2>/dev/null); do
- isExcluded "$x" && continue
+while read -r prog; do
+ isExcluded "$prog" && continue
if type -P flake8 &> /dev/null; then
- echo "flake8 $x"
- flake8 --max-line-length=120 --statistics $x
+ echo "flake8 $prog"
+ flake8 --max-line-length=120 --statistics "$prog"
echo; hr; echo
fi
for y in pyflakes pychecker; do
- if type -P $y &>/dev/null; then
- echo "$y $x"
- $y $x
+ if type -P "$y" &>/dev/null; then
+ echo "$y $prog"
+ "$y" "$prog"
echo; hr; echo
fi
done
-done
+done < <(find . -type f -name '*.py' -o -type f -name '*.jy')
diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py
new file mode 100755
index 000000000..4bd3aebc0
--- /dev/null
+++ b/tests/test_anonymize.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import os
+import re
+#import StringIO
+import subprocess
+from subprocess import PIPE
+import sys
+
+srcdir = os.path.abspath(os.path.dirname(__file__))
+
+anonymize_test_sh = os.path.join(srcdir, 'test_anonymize.sh')
+
+anonymize = '{}/../anonymize.py'.format(srcdir)
+
+src = {}
+dest = {}
+
+src_regex = re.compile(r'^\s*src\[(\d+)\]=["\'](.+)["\']\s*$')
+dest_regex = re.compile(r'^\s*dest\[(\d+)\]=["\'](.+)["\']\s*$')
+args_regex = re.compile(r'^\s*args=["\'](.+)["\']\s*$')
+
+def normalize_text(text):
+ text = text.replace(r'\"', '"')
+ text = text.replace(r"\'", "'")
+ return text
+
+def run():
+ #test_input = StringIO.StringIO()
+ #test_input.write('\n'.join(src))
+ global src # pylint: disable=global-statement
+ global dest # pylint: disable=global-statement
+ src = {int(k) : v for k, v in src.items()}
+ dest = {int(k) : v for k, v in dest.items()}
+ src_keys = sorted(src)
+ test_input = '\n'.join([src[_] for _ in src_keys])
+
+ print('running anonymize tests using: {} {}'.format(anonymize, args))
+ cmd = [anonymize] + args.split()
+ process = subprocess.Popen(cmd, stdin=PIPE, stdout=PIPE)
+ # encode as bytes for Python 3 :-/
+ test_input = str.encode(test_input, 'utf-8')
+ (stdout, _) = process.communicate(input=test_input)
+ index = 0
+ # convert bytes to string
+ stdout = stdout.decode("utf-8")
+ # pylint: disable=redefined-outer-name
+ for line in stdout.split('\n'):
+ key = src_keys[index]
+ _input = src[key]
+ expected = dest[key]
+ if line != expected:
+ print('FAILED to anonymize line during test {}'.format(key))
+ print('input: {}'.format(_input))
+ print('expected: {}'.format(expected))
+ print('got: {}'.format(line))
+ sys.exit(1)
+ print('SUCCEEDED anonymization test {}'.format(key))
+ index += 1
+
+with open(anonymize_test_sh) as filehandle:
+ for line in filehandle:
+ src_match = src_regex.match(line)
+ if src_match:
+ key = src_match.group(1)
+ if key in src:
+ raise AssertionError('Duplicate key index src[{}]'.format(key))
+ value = src_match.group(2)
+ value = normalize_text(value)
+ src[key] = value
+ dest_match = dest_regex.match(line)
+ if dest_match:
+ key = dest_match.group(1)
+ if key in dest:
+ raise AssertionError('Duplicate key index dest[{}]'.format(key))
+ value = dest_match.group(2)
+ value = normalize_text(value)
+ dest[key] = value
+ args_match = args_regex.match(line)
+ if args_match:
+ args = args_match.group(1)
+ run()
+ src = {}
+ dest = {}
diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh
index 20ff7fa24..9aebe794f 100755
--- a/tests/test_anonymize.sh
+++ b/tests/test_anonymize.sh
@@ -4,29 +4,30 @@
# Author: Hari Sekhon
# Date: 2015-07-28 18:47:41 +0100 (Tue, 28 Jul 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-test_nums="${@:-}"
-parallel=""
-if [ "$test_nums" = "p" ]; then
- parallel="1"
- test_nums=""
-fi
+test_nums="${*:-}"
+#parallel=""
+#if [ "$test_nums" = "p" ]; then
+# parallel="1"
+# test_nums=""
+#fi
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "Anonymize"
@@ -39,40 +40,6 @@ start_time="$(start_timer "$anonymize")"
# Custom Tests
# ============================================================================ #
-if [ -z "$test_nums" ]; then
- echo
- echo "Running Custom Tests:"
- echo
- echo "checking file args:"
- run++
- if [ `$anonymize -ae README.md | wc -l` -gt 100 ]; then
- echo "SUCCEEDED - anonymized README.md > 100 lines"
- else
- echo "FAILED - suspicious README.md file arg result came to <= 100 lines"
- exit 1
- fi
- hr
-
- run_grep "@" $anonymize --email <<< "hari@domain.com"
- run_grep "@" $anonymize -E <<< "hari@domain.com"
-
- run_grep ".1" $anonymize -a --ip-prefix <<< "4.3.2.1"
- run_grep ".1/" $anonymize --ip-prefix <<< "4.3.2.1/24"
- run_grep ".1" $anonymize --ip-prefix <<< "4.3.2.1"
- run_grep ".4" $anonymize --ip-prefix <<< "ip-1-2-3-4"
- run_grep "ip-1-2-3-4-5" $anonymize --ip-prefix <<< "ip-1-2-3-4-5"
- run_grep "dip-1-2-3-4" $anonymize --ip-prefix <<< "dip-1-2-3-4"
- run_grep "5.4.3.2.1" $anonymize --ip-prefix <<< "5.4.3.2.1"
- run_grep "log4j-1.2.3.4.jar" $anonymize --ip-prefix <<< "log4j-1.2.3.4.jar"
- run_grep "/usr/hdp/2.6.2.0-123" $anonymize --ip-prefix <<< "/usr/hdp/2.6.2.0-123"
-
- run_grep "^http://[a-f0-9]{12}:80/path$" $anonymize --hash-hostnames <<< "http://test.domain.com:80/path"
- run_grep '^\\\\[a-f0-9]{12}\\mydir$' $anonymize --hash-hostnames <<< '\\test.domain.com\mydir'
- run_grep '-host [a-f0-9]{12}' $anonymize --hash-hostnames <<< '-host blah'
-fi
-
-# ============================================================================ #
-
src[0]="2015-11-19 09:59:59,893 - Execution of 'mysql -u root --password=somep@ssword! -h myHost.internal -s -e \"select version();\"' returned 1. ERROR 2003 (HY000): Can't connect to MySQL server on 'host.domain.com' (111)"
dest[0]="2015-11-19 09:59:59,893 - Execution of 'mysql -u root --password= -h -s -e \"select version();\"' returned 1. ERROR 2003 (HY000): Can't connect to MySQL server on '' (111)"
@@ -398,9 +365,9 @@ src[102]="-Dhost.domain.com=blah"
dest[102]="-Dhost.domain.com=blah"
# check escape codes get stripped if present (eg. if piping from grep --color-yes)
-#src[88]="some^[[01;31m^[[Khost^[[m^[[Kname:443"
-#src[88]="some\e[01;31m\e[Khost\e[m\e[K:443"
-src[103]="$(echo somehost:443 | grep --color=yes host)"
+# breaks test_anonymize.py which doesn't eval this, so put it explicitly
+#src[103]="$(echo somehost:443 | grep --color=yes host)"
+src[103]="some[01;31m[Khost[m[K:443"
dest[103]=":443"
src[104]='..., "user": "blah", "group": "blah2", "host": "blah3", ...'
@@ -430,6 +397,106 @@ dest[111]="127.0.0.1"
src[112]="travis token: Abc123"
dest[112]="travis token: "
+src[113]="arn:aws:iam::123456789012:user/hari"
+dest[113]="arn:aws:iam:::user/"
+
+src[114]="arn:aws:iam::123456789012:group/hari"
+dest[114]="arn:aws:iam:::group/"
+
+src[115]="arn:aws:iam::123456789012:user/Development/product_1234/*"
+dest[115]="arn:aws:iam:::user//*"
+
+src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*"
+dest[116]="arn:aws:iam:::group//*"
+
+src[117]="arn:aws:iam::123456789012:group/Development/product_1234/*"
+dest[117]="arn:aws:iam:::group//*"
+
+src[118]="arn:aws:s3:::my_corporate_bucket/Development/*"
+dest[118]="arn:aws:s3:::*"
+
+src[119]="AKIAIOSFODNN7EXAMPLE"
+dest[119]=""
+
+src[120]="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+dest[120]=""
+
+src[121]="AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE"
+dest[121]=""
+
+src[122]="ASIAIOSFODNN7EXAMPLE"
+dest[122]=""
+
+src[123]="AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA=="
+dest[123]=""
+
+# security groups
+src[124]="sg-5f63c627"
+dest[124]=""
+
+src[125]="s3://myBucket/file.txt"
+dest[125]="s3:///file.txt"
+
+src[126]="aws rds create-db-instance --db-name myDB"
+dest[126]="aws rds create-db-instance --db-name "
+
+src[127]="aws rds modify-db-instance --db-instance-identifier myDBinstance"
+dest[127]="aws rds modify-db-instance --db-instance-identifier "
+
+src[128]="--master-user-password blah"
+dest[128]="--master-user-password "
+
+src[129]="--master-username first.last"
+dest[129]="--master-username "
+
+src[130]="--schema-name mySchema"
+dest[130]="--schema-name "
+
+src[131]="=arn:aws:acm:us-east-1:123456:certificate/abc-123"
+dest[131]="=arn:aws:acm:us-east-1::certificate/"
+
+src[132]="--key-name my-key"
+dest[132]="--key-name "
+
+src[133]="-private-key my-key"
+dest[133]="-private-key "
+
+src[134]="aws elasticache create-cache-cluster --cache-cluster-id myCluster"
+dest[134]="aws elasticache create-cache-cluster --cache-cluster-id "
+
+src[135]="subnet-abc12345"
+dest[135]=""
+
+src[136]="arn:aws:acm:us-east-1:123456:function:myFunction123:7"
+dest[136]="arn:aws:acm:us-east-1::function::7"
+
+src[137]="aws lambda update-function-code --function-name hari-test --zip-file fileb://myfunction.zip"
+dest[137]="aws lambda update-function-code --function-name --zip-file fileb://"
+
+# shellcheck disable=SC2016
+src[138]=' aws elb create-load-balancer --load-balancer-name "$lb_name" ...'
+dest[138]=' aws elb create-load-balancer --load-balancer-name ...'
+
+src[139]=' in column "blah" of table "blah2"'
+dest[139]=' in column "" of table ""'
+
+src[140]='ssh -i myKey -N -L 8888:ec2-1-2-3-4.eu-west-1.compute.amazonaws.com:8888 hadoop@ec2-1-2-3-4.eu-west-1.compute.amazonaws.com'
+# email anonymization applies before fqdn anonymization
+#dest[140]='ssh -i myKey -N -L 8888::8888 @'
+dest[140]='ssh -i myKey -N -L 8888::8888 @'
+
+# shellcheck disable=SC1117
+src[141]="Failed to open HDFS file hdfs://nameservice1/user/hive/warehouse/area_2/my_database_2.db/my_table_2/part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory"
+# shellcheck disable=SC1117
+dest[141]="Failed to open HDFS file hdfs:///user//warehouse/.db//part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory"
+
+src[142]="ERROR: AnalysisException: Failed to load metadata for table: 'myCustomerTable2'"
+dest[142]="ERROR: AnalysisException: Failed to load metadata for table: ''"
+
+src[143]="PS /pwd> Connect-AppVeyorToComputer -AppVeyorUrl https://ci.appveyor.com -ApiToken a12bcdef3a45b6cdefab"
+dest[143]="PS /pwd> Connect-AppVeyorToComputer -AppVeyorUrl https:// -ApiToken "
+
+
# TODO: move proxy hosts to host matches and re-enable
#src[103]="proxy blah port 8080"
#dest[103]="proxy port "
@@ -440,12 +507,16 @@ dest[112]="travis token: "
args="-aPe"
test_anonymize(){
run++
- src="$1"
- dest="$2"
+ # shellcheck disable=SC2178
+ local src="$1"
+ # shellcheck disable=SC2178
+ local dest="$2"
#[ -z "${src[$i]:-}" ] && { echo "skipping test $i..."; continue; }
# didn't work for \e escape codes for ANSI stripping test
#result="$(echo -e "$src" | $anonymize $args)"
+ # shellcheck disable=SC2128
result="$($anonymize $args <<< "$src")"
+ # shellcheck disable=SC2128
if grep -xFq -- "$dest" <<< "$result"; then
echo -n "SUCCEEDED anonymization test $i"
if [ -n "${SHOW_OUTPUT:-}" ]; then
@@ -477,26 +548,26 @@ fi
# this gives the number of elements and prevents testing the last element(s) if commenting something out in the middle
#for (( i = 0 ; i < ${#src[@]} ; i++ )); do
run_tests(){
- test_numbers="${@:-${!src[@]}}"
+ # expands to the list of indicies in the array, starting at zero - this is easier to work with that ${#src} which is a total
+ # that is off by one for index usage and doesn't support sparse arrays for any missing/disabled test indicies
+ test_numbers="${*:-${!src[*]}}"
for i in $test_numbers; do
[ -n "${src[$i]:-}" ] || { echo "code error: src[$i] not defined"; exit 1; }
[ -n "${dest[$i]:-}" ] || { echo "code error: dest[$i] not defined"; exit 1; }
- if [ -n "$parallel" ]; then
- test_anonymize "${src[$i]}" "${dest[$i]}" &
- else
- test_anonymize "${src[$i]}" "${dest[$i]}"
- fi
+ #test_anonymize "${src[$i]}" "${dest[$i]}"
+ run++
done
+ "$srcdir/test_anonymize.py"
}
-echo
-echo "Running Standard Tests with --all --skip-exceptions"
-echo
-run_tests # ignore_run_unqualified
+#echo
+#echo "Running Standard Tests with --all --skip-exceptions"
+#echo
+run_tests "$@" # ignore_run_unqualified
-echo
-echo "Running Tests preseving text without --network enabled:"
-echo
+#echo
+#echo "Running Tests preseving text without --network enabled:"
+#echo
# check normal don't strip these
src[901]="reading password from foo"
dest[901]="reading password from foo"
@@ -505,11 +576,11 @@ src[902]="some description = blah, module = foo"
dest[902]="some description = blah, module = foo"
args="-HKEiu"
-run_tests 901 902 # ignore_run_unqualified
+#run_tests 901 902 # ignore_run_unqualified
-echo
-echo "Running Network Specific Tests:"
-echo
+#echo
+#echo "Running Network Specific Tests:"
+#echo
# now check --network / --cisco / --juniper do strip these
src[903]="reading password from bar"
dest[903]="reading password "
@@ -518,19 +589,73 @@ src[904]="some description = blah, module=bar"
dest[904]="some description "
args="--network"
-run_tests 903 904 # ignore_run_unqualified
-
-if [ -n "$parallel" ]; then
- # can't trust exit code for parallel yet, only for quick local testing
- exit 1
-# for i in ${!src[@]}; do
-# let j=$i+1
-# wait %$j
-# [ $? -eq 0 ] || { echo "FAILED"; exit $?; }
-# done
+#run_tests 903 904 # ignore_run_unqualified
+
+#if [ -n "$parallel" ]; then
+# # can't trust exit code for parallel yet, only for quick local testing
+# exit 1
+## for i in ${!src[@]}; do
+## let j=$i+1
+## wait %$j
+## [ $? -eq 0 ] || { echo "FAILED"; exit $?; }
+## done
+#fi
+
+# ============================================================================ #
+
+if [ -z "$test_nums" ]; then
+ echo
+ echo "Running Custom Tests:"
+ echo
+ echo "checking file args:"
+ run++
+ if [ "$($anonymize -ae README.md | wc -l)" -gt 100 ]; then
+ echo "SUCCEEDED - anonymized README.md > 100 lines"
+ else
+ echo "FAILED - suspicious README.md file arg result came to <= 100 lines"
+ exit 1
+ fi
+ hr
+
+ run_grep "@" $anonymize --email <<< "hari@domain.com"
+ run_grep "@" $anonymize -E <<< "hari@domain.com"
+
+ src[800]="4.3.2.1"
+ dest[800]=".1"
+
+ src[801]="4.3.2.1/24"
+ dest[801]=".1/"
+
+ src[802]="4.3.2.1"
+ dest[802]=".1"
+
+ src[803]="ip-1-2-3-4"
+ dest[803]=".4"
+
+ src[804]="ip-1-2-3-4-5"
+ dest[804]="ip-1-2-3-4-5"
+
+ src[805]="dip-1-2-3-4"
+ dest[805]="dip-1-2-3-4"
+
+ src[806]="5.4.3.2.1"
+ dest[806]="5.4.3.2.1"
+
+ src[807]="log4j-1.2.3.4.jar"
+ dest[807]="log4j-1.2.3.4.jar"
+
+ src[808]="/usr/hdp/2.6.2.0-123"
+ dest[808]="/usr/hdp/2.6.2.0-123"
+ args="-a --ip-prefix"
+
+ run_grep "^http://[a-f0-9]{12}:80/path$" $anonymize --hash-hostnames <<< "http://test.domain.com:80/path"
+ run_grep '^\\\\[a-f0-9]{12}\\mydir$' $anonymize --hash-hostnames <<< '\\test.domain.com\mydir'
+ run_grep '-host [a-f0-9]{12}' $anonymize --hash-hostnames <<< '-host blah'
fi
echo
+# run_count assigned in utils lib
+# shellcheck disable=SC2154
echo "Total Tests run: $run_count"
time_taken "$start_time" "SUCCESS! All tests for $anonymize completed in"
echo
diff --git a/tests/test_apache-drill.sh b/tests/test_apache-drill.sh
index 03c6539a8..af4b663c6 100755
--- a/tests/test_apache-drill.sh
+++ b/tests/test_apache-drill.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-26 23:36:03 +0000 (Tue, 26 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -20,11 +20,12 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/.."
+# shellcheck disable=SC1090
. "$srcdir/utils.sh"
section "A p a c h e D r i l l"
-export APACHE_DRILL_VERSIONS="${@:-${APACHE_DRILL_VERSIONS:-0.7 0.8 0.9 1.0 1.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 1.12 1.13 1.14 1.15 latest}}"
+export APACHE_DRILL_VERSIONS="${*:-${APACHE_DRILL_VERSIONS:-0.7 0.8 0.9 1.0 1.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 1.12 1.13 1.14 1.15 latest}}"
APACHE_DRILL_HOST="${DOCKER_HOST:-${APACHE_DRILL_HOST:-${HOST:-localhost}}}"
APACHE_DRILL_HOST="${APACHE_DRILL_HOST##*/}"
@@ -45,6 +46,7 @@ test_apache_drill(){
echo "getting Apache Drill dynamic port mappings:"
docker_compose_port "Apache Drill"
hr
+ # shellcheck disable=SC2153
when_ports_available "$APACHE_DRILL_HOST" "$APACHE_DRILL_PORT"
hr
when_url_content "http://$APACHE_DRILL_HOST:$APACHE_DRILL_PORT/status" "Running"
@@ -60,15 +62,19 @@ test_apache_drill(){
hr
APACHE_DRILL_PORT="$APACHE_DRILL_PORT_DEFAULT" ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_apache_drill.py $non_drill_node1 $non_drill_node2
+ # shellcheck disable=SC2097,SC2098
APACHE_DRILL_PORT="$APACHE_DRILL_PORT_DEFAULT" run_grep "^$APACHE_DRILL_HOST:$APACHE_DRILL_PORT$" ./find_active_apache_drill.py $non_drill_node1 "$APACHE_DRILL_HOST:$APACHE_DRILL_PORT"
# Drill 1.7+ only
- if [ "$version" = "latest" ] || [[ "$version" > 1.6 ]]; then
+ if [ "$version" = "latest" ] || [ "$(bc <<< "$version > 1.6")" = 1 ]; then
APACHE_DRILL_PORT="$APACHE_DRILL_PORT_DEFAULT" ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_apache_drill2.py $non_drill_node1 $non_drill_node2
+ # shellcheck disable=SC2097,SC2098
APACHE_DRILL_PORT="$APACHE_DRILL_PORT_DEFAULT" run_grep "^$APACHE_DRILL_HOST:$APACHE_DRILL_PORT$" ./find_active_apache_drill2.py $non_drill_node1 "$APACHE_DRILL_HOST:$APACHE_DRILL_PORT"
fi
+ # run_count defined in util lib
+ # shellcheck disable=SC2154
echo "Completed $run_count Apache Drill tests"
hr
[ -n "${KEEPDOCKER:-}" ] ||
@@ -76,6 +82,6 @@ test_apache_drill(){
echo
}
-startupwait 70
+startupwait 120
run_test_versions "Apache Drill"
diff --git a/tests/test_center.sh b/tests/test_center.sh
index 976466e1a..adb080442 100755
--- a/tests/test_center.sh
+++ b/tests/test_center.sh
@@ -10,7 +10,7 @@
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# Quick tests, need to replace with testcmd.exp
@@ -22,6 +22,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/.."
+# shellcheck disable=SC1091
. "bash-tools/lib/utils.sh"
section "Testing center.py"
@@ -127,6 +128,8 @@ echo "testing spacing with stdin:"
run_output "$expected" ./center.py -s <<< " "
echo
+# $run_count defined in lib
+# shellcheck disable=SC2154
echo "Completed $run_count tests"
echo
echo "All tests for center.py completed successfully"
diff --git a/tests/test_docker.sh b/tests/test_docker.sh
index ddb02baa7..37dbb90d5 100755
--- a/tests/test_docker.sh
+++ b/tests/test_docker.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-12-08 14:38:37 +0000 (Thu, 08 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,7 +19,10 @@ srcdir="$(cd "$(dirname "$0")" && pwd)"
cd "$srcdir/.."
+# shellcheck disable=SC1091
. "bash-tools/lib/docker.sh"
+
+# shellcheck disable=SC1091
. "bash-tools/lib/utils.sh"
section "Docker Image"
diff --git a/tests/test_docker_registry_show_tags.sh b/tests/test_docker_registry_show_tags.sh
index d3d72b932..e327a3135 100755
--- a/tests/test_docker_registry_show_tags.sh
+++ b/tests/test_docker_registry_show_tags.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-09-12 17:49:29 +0200 (Tue, 12 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,8 +19,10 @@ srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir2/.."
+# shellcheck disable=SC1091
. "tests/utils.sh"
+# shellcheck disable=SC1091
. "bash-tools/lib/utils.sh"
srcdir="$srcdir2"
@@ -75,12 +77,13 @@ private_key="registry.key"
certificate="registry.crt"
htpasswd="registry.htpasswd"
csr="registry.csr"
-if ! [ -f "$private_key" -a -f "$certificate" ]; then
+if ! [ -f "$private_key" ] &&
+ [ -f "$certificate" ]; then
echo "Generating sample SSL certificates:"
echo
openssl genrsa -out "$private_key" 2048
echo
- yes "" | openssl req -new -key "$private_key" -out "$csr" || :
+ yes "." | openssl req -new -key "$private_key" -out "$csr" || :
echo
openssl x509 -req -days 3650 -in "$csr" -signkey "$private_key" -out "$certificate"
echo
@@ -108,6 +111,7 @@ echo "getting dynamic Docker Registry port mapping:"
docker_compose_port "Docker Registry"
hr
+# shellcheck disable=SC2153
if [ -z "$DOCKER_REGISTRY_PORT" ]; then
echo "DOCKER_REGISTRY_PORT not found from running container, did container fail to start up properly?"
exit 1
diff --git a/tests/test_dockerfiles_check_git_branches.sh b/tests/test_dockerfiles_check_git_branches.sh
index ec2be2a11..1081c3b98 100755
--- a/tests/test_dockerfiles_check_git_branches.sh
+++ b/tests/test_dockerfiles_check_git_branches.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,15 +19,17 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/.."
+# shellcheck disable=SC1091
. "tests/utils.sh"
+# shellcheck disable=SC1091
. "bash-tools/lib/utils.sh"
section "Testing Dockerfiles Check Git branches"
if type -P git &>dev/null; then
if ! [ -d Dockerfiles ]; then
- git clone https://github.com/harisekhon/Dockerfiles
+ git clone https://github.com/HariSekhon/Dockerfiles
else
pushd Dockerfiles
git pull
diff --git a/tests/test_dockerfiles_check_git_tags.sh b/tests/test_dockerfiles_check_git_tags.sh
index 33556738c..a6e0ba12c 100755
--- a/tests/test_dockerfiles_check_git_tags.sh
+++ b/tests/test_dockerfiles_check_git_tags.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,15 +19,17 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/.."
+# shellcheck disable=SC1091
. "tests/utils.sh"
+# shellcheck disable=SC1091
. "bash-tools/lib/utils.sh"
section "Testing Dockerfiles Check Git Tags"
if type -P git &>dev/null; then
if ! [ -d Dockerfiles ]; then
- git clone https://github.com/harisekhon/Dockerfiles
+ git clone https://github.com/HariSekhon/Dockerfiles
else
pushd Dockerfiles
git pull
diff --git a/tests/test_dockerhub_search.sh b/tests/test_dockerhub_search.sh
index d625bf2db..2a4f9c48e 100755
--- a/tests/test_dockerhub_search.sh
+++ b/tests/test_dockerhub_search.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,19 +19,24 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/.."
+# shellcheck disable=SC1091
. "tests/utils.sh"
+# shellcheck disable=SC1091
. "bash-tools/lib/utils.sh"
section "Testing DockerHub Show Tags"
check './dockerhub_search.py centos' "DockerHub Search for CentOS"
check './dockerhub_search.py harisekhon' "DockerHub Search for harisekhon"
-check './dockerhub_search.py harisekhon -n 30' "DockerHub Search for harisekhon -n 30"
-check './dockerhub_search.py harisekhon/hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev"
+check './dockerhub_search.py harisekhon -l 30' "DockerHub Search for harisekhon -l 30"
+# this no longer works, API must have changed
+#check './dockerhub_search.py harisekhon/hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev"
+check './dockerhub_search.py hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev"
# causes IOError: [Errno 32] Broken pipe
#unset PYTHONUNBUFFERED
-check '[ $(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep "^harisekhon/[A-Za-z0-9_-]*$" | wc -l) = 40 ]' "DockerHub Search quiet mode for shell scripting"
+# shellcheck disable=SC2016
+check '[ "$(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep -c "^harisekhon/[A-Za-z0-9_-]*$")" = 40 ]' "DockerHub Search quiet mode for shell scripting"
echo
echo
diff --git a/tests/test_dockerhub_show_tags.sh b/tests/test_dockerhub_show_tags.sh
index 71f424a3f..dba64e816 100755
--- a/tests/test_dockerhub_show_tags.sh
+++ b/tests/test_dockerhub_show_tags.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/.."
+# shellcheck disable=SC1091
. "tests/utils.sh"
+# shellcheck disable=SC1091
. "bash-tools/lib/utils.sh"
section "Testing DockerHub Show Tags"
@@ -39,6 +41,8 @@ echo
echo
echo "All DockerHub Show Tags tests completed successfully"
echo
+# $run_count defined in lib
+# shellcheck disable=SC2154
echo "Total Tests run: $run_count"
time_taken "$start_time" "DockerHub Show Tags tests completed in"
echo
diff --git a/tests/test_elasticsearch.sh b/tests/test_elasticsearch.sh
index e18fd8d09..17cd5260d 100755
--- a/tests/test_elasticsearch.sh
+++ b/tests/test_elasticsearch.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,11 +19,12 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "E l a s t i c s e a r c h"
-export ELASTICSEARCH_VERSIONS="${@:-${ELASTICSEARCH_VERSIONS:-latest 1.3 1.4 1.5 1.6 1.7 2.0 2.1 2.2 2.3 2.4 5.0 5.1 5.2 5.3 5.4 5.5 5.6 6.0.1 6.1.1}}"
+export ELASTICSEARCH_VERSIONS="${*:-${ELASTICSEARCH_VERSIONS:-latest 1.3 1.4 1.5 1.6 1.7 2.0 2.1 2.2 2.3 2.4 5.0 5.1 5.2 5.3 5.4 5.5 5.6 6.0.1 6.1.1}}"
ELASTICSEARCH_HOST="${DOCKER_HOST:-${ELASTICSEARCH_HOST:-${HOST:-localhost}}}"
ELASTICSEARCH_HOST="${ELASTICSEARCH_HOST##*/}"
@@ -43,7 +44,7 @@ test_elasticsearch(){
local version="$1"
section2 "Setting up Elasticsearch $version test container"
if [ "$version" != "latest" ] && [ "${version:0:1}" -ge 6 ]; then
- local export COMPOSE_FILE="$srcdir/docker/$DOCKER_SERVICE-elastic.co-docker-compose.yml"
+ export COMPOSE_FILE="$srcdir/docker/$DOCKER_SERVICE-elastic.co-docker-compose.yml"
fi
docker_compose_pull
VERSION="$version" docker-compose up -d
@@ -63,6 +64,7 @@ test_elasticsearch(){
ELASTICSEARCH_PORT="$ELASTICSEARCH_PORT_DEFAULT" \
ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_elasticsearch.py $non_es_node1 $non_es_node2
+ # shellcheck disable=SC2097,SC2098
ELASTICSEARCH_PORT="$ELASTICSEARCH_PORT_DEFAULT" \
run_grep "^$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT$" ./find_active_elasticsearch.py $non_es_node1 $non_es_node2 "$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT"
diff --git a/tests/test_find_active_server.sh b/tests/test_find_active_server.sh
index 74bb8e494..615f806d4 100755
--- a/tests/test_find_active_server.sh
+++ b/tests/test_find_active_server.sh
@@ -10,7 +10,7 @@
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -20,6 +20,7 @@ srcdir="$(cd "$(dirname "$0")" && pwd)"
cd "$srcdir/.."
+# shellcheck disable=SC1091
. bash-tools/lib/utils.sh
set +e +o pipefail
@@ -143,7 +144,7 @@ ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_server.py --https local
echo "testing https with url path and regex matching:"
echo
-run_grep "^github.com$" ./find_active_server.py $opts --https $WEBSITE2 github.com -u /harisekhon --regex 'python-tools'
+run_grep "^github.com$" ./find_active_server.py $opts --https $WEBSITE2 github.com -u /harisekhon --regex '(?i)python-tools'
# ============================================================================ #
@@ -174,19 +175,19 @@ count_socket_attempts=0
found_google_socket=0
found_duckduckgo_socket=0
run++
-for x in {1..10}; do
+for _ in {1..10}; do
echo -n .
- let count_socket_attempts+=1
+ ((count_socket_attempts+=1))
output="$(./find_active_server.py -n1 --random --port 80 $WEBSITE1 $WEBSITE2)"
if [ "$output" = "$WEBSITE2" ]; then
found_google_socket=1
elif [ "$output" = "$WEBSITE1" ]; then
found_duckduckgo_socket=1
fi
- [ $found_google_socket -eq 1 -a $found_duckduckgo_socket -eq 1 ] && break
+ [ $found_google_socket -eq 1 ] && [ $found_duckduckgo_socket -eq 1 ] && break
done
echo
-if [ $found_google_socket -eq 1 -a $found_duckduckgo_socket -eq 1 ]; then
+if [ $found_google_socket -eq 1 ] && [ $found_duckduckgo_socket -eq 1 ]; then
echo "Found both $WEBSITE1 and $WEBSITE2 in results from $count_socket_attempts --random runs"
else
die "Failed to return both $WEBSITE1 and $WEBSITE2 in results from $count_socket_attempts --random runs"
@@ -210,25 +211,27 @@ count_http_attempts=0
found_google_http=0
found_duckduckgo_http=0
run++
-for x in {1..10}; do
+for _ in {1..10}; do
echo -n .
- let count_http_attempts+=1
+ ((count_http_attempts+=1))
output="$(./find_active_server.py -n1 --http --random $WEBSITE1 $WEBSITE2)"
if [ "$output" = "$WEBSITE2" ]; then
found_google_http=1
elif [ "$output" = "$WEBSITE1" ]; then
found_duckduckgo_http=1
fi
- [ $found_google_http -eq 1 -a $found_duckduckgo_http -eq 1 ] && break
+ [ $found_google_http -eq 1 ] && [ $found_duckduckgo_http -eq 1 ] && break
done
echo
-if [ $found_google_http -eq 1 -a $found_duckduckgo_http -eq 1 ]; then
+if [ $found_google_http -eq 1 ] && [ $found_duckduckgo_http -eq 1 ]; then
echo "Found both $WEBSITE1 and $WEBSITE2 in results from $count_http_attempts --random runs"
else
die "Failed to return both $WEBSITE1 and $WEBSITE2 in results from $count_http_attempts --random runs"
fi
hr
echo
+# $run_count defined in lib
+# shellcheck disable=SC2154
echo "Tests run: $run_count"
time_taken "$start_time" "find_active_server.py tests completed in"
echo
diff --git a/tests/test_find_duplicate_files.sh b/tests/test_find_duplicate_files.sh
index 807508f52..ade1eeb00 100755
--- a/tests/test_find_duplicate_files.sh
+++ b/tests/test_find_duplicate_files.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-08-14 20:42:01 +0100 (Sun, 14 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -20,6 +20,7 @@ srcdir="$(cd "$(dirname "$0")" && pwd)"
cd "$srcdir/.."
+# shellcheck disable=SC1091
. "bash-tools/lib/utils.sh"
section "find_duplicate_files.py"
@@ -29,6 +30,7 @@ start_time="$(start_timer "find_duplicate_files.py test")"
testdir1="$(cd tests/data/ && mktemp -d -t tmp_find_duplicate_files.XXXXXX)"
testdir2="$(cd tests/data/ && mktemp -d -t tmp_find_duplicate_files2.XXXXXX)"
+# shellcheck disable=SC2064,SC2086
trap "rm -fr '$testdir1' '$testdir2'" $TRAP_SIGNALS
echo test > "$testdir1/test1.txt"
@@ -108,10 +110,10 @@ for testdir in "$testdir1" "$testdir2"; do
echo "now check the file basename matches on 'est'":
run_fail 4 ./find_duplicate_files.py --regex 'est' "$testdir" "$testdir1" --quiet
- echo "now check the file basename matches with specified capture subset '(est)\d'":
+ echo "now check the file basename matches with specified capture subset '(est)\\d'":
run_fail 4 ./find_duplicate_files.py --regex '(est)\d' "$testdir" "$testdir1"
- echo "now check the file basename doesn't match when the capture includes differing numbers 'est\d'":
+ echo "now check the file basename doesn't match when the capture includes differing numbers 'est\\d'":
run ./find_duplicate_files.py --regex 'est\d' "$testdir" "$testdir1"
rm "$testdir/test2.txt"
@@ -135,6 +137,8 @@ rm -fr "$testdir1" "$testdir2"
echo
echo
+# $run_count defined in lib
+# shellcheck disable=SC2154
echo "Tests run: $run_count"
time_taken "$start_time" "find_duplicate_files.py tests completed in"
echo
diff --git a/tests/test_find_python_library_path.sh b/tests/test_find_python_library_path.sh
index 6a13f5779..7cdc1f43f 100755
--- a/tests/test_find_python_library_path.sh
+++ b/tests/test_find_python_library_path.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2019-09-27
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/tests/test_getent.sh b/tests/test_getent.sh
index d2b87ddfe..6afc48810 100755
--- a/tests/test_getent.sh
+++ b/tests/test_getent.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-11-20 15:35:37 +0000 (Sun, 20 Nov 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,34 +19,38 @@ srcdir="$(cd "$(dirname "$0")" && pwd)"
cd "$srcdir/.."
+# shellcheck disable=SC1091
. ./bash-tools/lib/utils.sh
section "Getent"
start_time="$(start_timer "find_active_server.py test")"
-system=`uname -s`
+system="$(uname -s)"
echo "system = $system"
hr
-if [ "$system" = "Linux" -o "$system" = Darwin ]; then
+if [ "$system" = "Linux" ] ||
+ [ "$system" = Darwin ]; then
run ./getent.py passwd | grep -v ':[x*!]*:'
# counter is lost in subshell, increment manually
run++
# $USER isn't always available in docker containers, use 'id' instead
- run ./getent.py passwd `id -un`
+ run ./getent.py passwd "$(id -un)"
run_fail 2 ./getent.py passwd nonexistentuser
run ./getent.py group | grep -v ':[x*!]:'
run++
- run ./getent.py group `id -gn`
+ run ./getent.py group "$(id -gn)"
run_fail 2 ./getent.py group nonexistentgroup
echo
+ # $run_count defined in lib
+ # shellcheck disable=SC2154
echo "Tests run: $run_count"
time_taken "$start_time" "find_active_server.py tests completed in"
else
diff --git a/tests/test_git_check_branches_upstream.sh b/tests/test_git_check_branches_upstream.sh
index 878529a01..8459249bc 100755
--- a/tests/test_git_check_branches_upstream.sh
+++ b/tests/test_git_check_branches_upstream.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,15 +19,17 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/.."
+# shellcheck disable=SC1091
. "tests/utils.sh"
+# shellcheck disable=SC1091
. "bash-tools/lib/utils.sh"
section "Testing Git check branches upstream"
if type -P git &>/dev/null; then
if ! [ -d Dockerfiles ]; then
- git clone https://github.com/harisekhon/Dockerfiles
+ git clone https://github.com/HariSekhon/Dockerfiles
else
pushd Dockerfiles
git pull
diff --git a/tests/test_hadoop.sh b/tests/test_hadoop.sh
index 25b705f8a..479789e1f 100755
--- a/tests/test_hadoop.sh
+++ b/tests/test_hadoop.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-05-06 12:12:15 +0100 (Fri, 06 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,12 +19,13 @@ srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$srcdir/.."
+# shellcheck disable=SC1090
. "$srcdir/utils.sh"
section "H a d o o p"
# find_active_hadoop_namenode.py doesn't work on Hadoop 2.2 as the JMX bean isn't present
-export HADOOP_VERSIONS="${@:-${HADOOP_VERSIONS:-latest 2.3 2.4 2.5 2.6 2.7 2.8}}"
+export HADOOP_VERSIONS="${*:-${HADOOP_VERSIONS:-latest 2.3 2.4 2.5 2.6 2.7 2.8}}"
HADOOP_HOST="${DOCKER_HOST:-${HADOOP_HOST:-${HOST:-localhost}}}"
HADOOP_HOST="${HADOOP_HOST##*/}"
@@ -61,6 +62,7 @@ test_hadoop(){
docker_compose_port HADOOP_YARN_NODE_MANAGER_PORT "Yarn NM"
export HADOOP_PORTS="$HADOOP_NAMENODE_PORT $HADOOP_DATANODE_PORT $HADOOP_YARN_RESOURCE_MANAGER_PORT $HADOOP_YARN_NODE_MANAGER_PORT"
hr
+ # shellcheck disable=SC2086
when_ports_available "$HADOOP_HOST" $HADOOP_PORTS
hr
# don't use the worker nodes so not testing for their availability
@@ -125,10 +127,12 @@ EOFCOMMENTED
# therefore reset the HADOOP PORTS to point to something that should get connection refused like port 1 and so that the failure hosts still fail and return only the expected correct host
HADOOP_NAMENODE_PORT=1 ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_hadoop_namenode.py 127.0.0.2 127.0.0.3 "$HADOOP_HOST:$HADOOP_DATANODE_PORT"
+ # shellcheck disable=SC2097,SC2098
HADOOP_NAMENODE_PORT=1 run_grep "^$HADOOP_HOST:$HADOOP_NAMENODE_PORT$" ./find_active_hadoop_namenode.py 127.0.0.2 "$HADOOP_HOST:$HADOOP_DATANODE_PORT" 127.0.0.3 "$HADOOP_HOST:$HADOOP_NAMENODE_PORT"
HADOOP_YARN_RESOURCE_MANAGER_PORT=1 ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_hadoop_yarn_resource_manager.py 127.0.0.2 127.0.0.3 "$HADOOP_HOST:$HADOOP_YARN_NODE_MANAGER_PORT"
+ # shellcheck disable=SC2097,SC2098
HADOOP_YARN_RESOURCE_MANAGER_PORT=1 run_grep "^$HADOOP_HOST:$HADOOP_YARN_RESOURCE_MANAGER_PORT$" ./find_active_hadoop_yarn_resource_manager.py 127.0.0.2 "$HADOOP_HOST:$HADOOP_YARN_NODE_MANAGER_PORT" 127.0.0.3 "$HADOOP_HOST:$HADOOP_YARN_RESOURCE_MANAGER_PORT"
[ -z "${KEEPDOCKER:-}" ] ||
docker-compose down
diff --git a/tests/test_hbase.sh b/tests/test_hbase.sh
index ebde33a96..4727347a6 100755
--- a/tests/test_hbase.sh
+++ b/tests/test_hbase.sh
@@ -4,26 +4,27 @@
# Author: Hari Sekhon
# Date: 2016-05-06 12:12:15 +0100 (Fri, 06 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
-srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-cd "$srcdir2/.."
+cd "$srcdir/.."
-. "$srcdir2/utils.sh"
-. "$srcdir2/../bash-tools/lib/docker.sh"
+# shellcheck disable=SC1090
+. "$srcdir/utils.sh"
-srcdir="$srcdir2"
+# shellcheck disable=SC1090
+. "$srcdir/../bash-tools/lib/docker.sh"
section "H B a s e"
@@ -39,7 +40,7 @@ export HBASE_THRIFT_PORT_DEFAULT=9090
export HBASE_THRIFT_UI_PORT_DEFAULT=9095
export ZOOKEEPER_PORT_DEFAULT=2181
-export HBASE_VERSIONS="${@:-latest 0.96 0.98 1.0 1.1 1.2 1.3}"
+export HBASE_VERSIONS="${*:-latest 0.96 0.98 1.0 1.1 1.2 1.3}"
check_docker_available
@@ -59,9 +60,10 @@ test_hbase(){
fi
VERSION="$version" docker-compose up -d
hr
- if [ "$version" = "0.96" -o "$version" = "0.98" ]; then
- local export HBASE_MASTER_PORT_DEFAULT=60010
- local export HBASE_REGIONSERVER_PORT_DEFAULT=60301
+ if [ "$version" = "0.96" ] ||
+ [ "$version" = "0.98" ]; then
+ export HBASE_MASTER_PORT_DEFAULT=60010
+ export HBASE_REGIONSERVER_PORT_DEFAULT=60301
fi
echo "getting HBase dynamic port mappings:"
docker_compose_port "HBase Master"
@@ -73,6 +75,7 @@ test_hbase(){
#docker_compose_port ZOOKEEPER_PORT "HBase ZooKeeper"
export HBASE_PORTS="$HBASE_MASTER_PORT $HBASE_REGIONSERVER_PORT $HBASE_STARGATE_PORT $HBASE_STARGATE_UI_PORT $HBASE_THRIFT_PORT $HBASE_THRIFT_UI_PORT"
hr
+ # shellcheck disable=SC2086
when_ports_available "$HBASE_HOST" $HBASE_PORTS
hr
if [ "${version:0:3}" = "0.9" ]; then
@@ -118,10 +121,12 @@ EOF
return
fi
# will otherwise pick up HBASE_HOST and use default port and return the real HBase Master
+ # shellcheck disable=SC2097,SC2098
HBASE_HOST='' HOST='' HBASE_MASTER_PORT="$HBASE_MASTER_PORT_DEFAULT" \
ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_hbase_master.py 127.0.0.2 127.0.0.3 "$HBASE_HOST:$HBASE_REGIONSERVER_PORT"
# if HBASE_PORT / --port is set to same as suffix then only outputs host not host:port
+ # shellcheck disable=SC2097,SC2098
HBASE_HOST='' HOST='' HBASE_MASTER_PORT="$HBASE_MASTER_PORT_DEFAULT" \
run_grep "^$HBASE_HOST:$HBASE_MASTER_PORT$" ./find_active_hbase_master.py 127.0.0.2 "$HBASE_HOST:$HBASE_REGIONSERVER_PORT" 127.0.0.3 "$HBASE_HOST:$HBASE_MASTER_PORT"
@@ -245,53 +250,53 @@ EOF
run_conn_refused ./hbase_table_row_key_distribution.py -T HexStringSplitTable
# ============================================================================ #
- run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 2
- run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 2 --average
- run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 2 --average --skip-zeros
+ run ./hbase_region_requests.py -T HexStringSplitTable "$HBASE_HOST" -c 2
+ run ./hbase_region_requests.py -T HexStringSplitTable "$HBASE_HOST" -c 2 --average
+ run ./hbase_region_requests.py -T HexStringSplitTable "$HBASE_HOST" -c 2 --average --skip-zeros
- run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 2
- run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 2 --skip-zeros
- run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 2 --average
+ run ./hbase_region_requests.py -T HS_test_data "$HBASE_HOST" -c 2
+ run ./hbase_region_requests.py -T HS_test_data "$HBASE_HOST" -c 2 --skip-zeros
+ run ./hbase_region_requests.py -T HS_test_data "$HBASE_HOST" -c 2 --average
- run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST --count 2 --interval 2
+ run ./hbase_region_requests.py -T HS_test_data "$HBASE_HOST" --count 2 --interval 2
- run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST -c 2
- run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST --count 2 -i 2
- run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST -c 2 --average
+ run ./hbase_region_requests.py -T HS_test_data localhost "$HBASE_HOST" -c 2
+ run ./hbase_region_requests.py -T HS_test_data localhost "$HBASE_HOST" --count 2 -i 2
+ run ./hbase_region_requests.py -T HS_test_data localhost "$HBASE_HOST" -c 2 --average
# ============================================================================ #
- run ./hbase_regionserver_requests.py $HBASE_HOST -c 1
- run ./hbase_regionserver_requests.py $HBASE_HOST -c 1 --average
+ run ./hbase_regionserver_requests.py "$HBASE_HOST" -c 1
+ run ./hbase_regionserver_requests.py "$HBASE_HOST" -c 1 --average
- run ./hbase_regionserver_requests.py $HBASE_HOST -c 1 -T read,write,total
- run ./hbase_regionserver_requests.py $HBASE_HOST -c 1 --type read,write,total --average
+ run ./hbase_regionserver_requests.py "$HBASE_HOST" -c 1 -T read,write,total
+ run ./hbase_regionserver_requests.py "$HBASE_HOST" -c 1 --type read,write,total --average
- run ./hbase_regionserver_requests.py $HBASE_HOST --count 2 --interval 2
+ run ./hbase_regionserver_requests.py "$HBASE_HOST" --count 2 --interval 2
- run ./hbase_regionserver_requests.py localhost $HBASE_HOST -c 1
- run ./hbase_regionserver_requests.py localhost $HBASE_HOST --count 2 -i 2
- run ./hbase_regionserver_requests.py localhost $HBASE_HOST -c 1 --average
+ run ./hbase_regionserver_requests.py localhost "$HBASE_HOST" -c 1
+ run ./hbase_regionserver_requests.py localhost "$HBASE_HOST" --count 2 -i 2
+ run ./hbase_regionserver_requests.py localhost "$HBASE_HOST" -c 1 --average
# ============================================================================ #
- run ./hbase_regions_by_size.py $HBASE_HOST
- run ./hbase_regions_by_size.py $HBASE_HOST --smallest
- run ./hbase_regions_by_size.py $HBASE_HOST --human
- run ./hbase_regions_by_size.py $HBASE_HOST --human -s
- run ./hbase_regions_by_size.py $HBASE_HOST --human --top 10
- run ./hbase_regions_by_size.py $HBASE_HOST --human --top 10 --smallest
-
- run ./hbase_regions_by_memstore_size.py $HBASE_HOST
- run ./hbase_regions_by_memstore_size.py $HBASE_HOST --smallest
- run ./hbase_regions_by_memstore_size.py $HBASE_HOST --human
- run ./hbase_regions_by_memstore_size.py $HBASE_HOST --human -s
- run ./hbase_regions_by_memstore_size.py $HBASE_HOST --human --top 10
- run ./hbase_regions_by_memstore_size.py $HBASE_HOST --human --top 10 --smallest
+ run ./hbase_regions_by_size.py "$HBASE_HOST"
+ run ./hbase_regions_by_size.py "$HBASE_HOST" --smallest
+ run ./hbase_regions_by_size.py "$HBASE_HOST" --human
+ run ./hbase_regions_by_size.py "$HBASE_HOST" --human -s
+ run ./hbase_regions_by_size.py "$HBASE_HOST" --human --top 10
+ run ./hbase_regions_by_size.py "$HBASE_HOST" --human --top 10 --smallest
+
+ run ./hbase_regions_by_memstore_size.py "$HBASE_HOST"
+ run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --smallest
+ run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --human
+ run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --human -s
+ run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --human --top 10
+ run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --human --top 10 --smallest
# ============================================================================ #
- run ./hbase_regions_least_used.py $HBASE_HOST -r 20000
- run ./hbase_regions_least_used.py $HBASE_HOST -r 0
- run ./hbase_regions_least_used.py $HBASE_HOST --human --requests 20000
- run ./hbase_regions_least_used.py $HBASE_HOST --human --requests 20000 --top 10
+ run ./hbase_regions_least_used.py "$HBASE_HOST" -r 20000
+ run ./hbase_regions_least_used.py "$HBASE_HOST" -r 0
+ run ./hbase_regions_least_used.py "$HBASE_HOST" --human --requests 20000
+ run ./hbase_regions_least_used.py "$HBASE_HOST" --human --requests 20000 --top 10
[ -z "${KEEPDOCKER:-}" ] ||
docker-compose down
diff --git a/tests/test_headtail.sh b/tests/test_headtail.sh
index 2ebb80933..2dcfb599c 100755
--- a/tests/test_headtail.sh
+++ b/tests/test_headtail.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -25,6 +25,7 @@ echo "
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
until [ $# -lt 1 ]; do
@@ -34,7 +35,7 @@ until [ $# -lt 1 ]; do
done
data_dir="tests/data"
-broken_dir="$data_dir/broken_json_data"
+#broken_dir="$data_dir/broken_json_data"
testfile="$data_dir/plant_catalog.xml"
@@ -42,7 +43,7 @@ check(){
cmd="$1"
expected="$2"
msg="$3"
- output="$(eval $cmd)"
+ output="$(eval "$cmd")"
result="$(cksum <<< "$output")"
echo -n "checking headtail $msg => "
if [ "$result" = "$expected" ]; then
@@ -52,7 +53,7 @@ check(){
echo
echo "full output: "
echo
- eval $cmd
+ eval "$cmd"
echo
echo "cksum: $result"
exit 1
diff --git a/tests/test_hexanonymize.sh b/tests/test_hexanonymize.sh
new file mode 100755
index 000000000..9477001b2
--- /dev/null
+++ b/tests/test_hexanonymize.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-01-02 17:35:08 +0000 (Thu, 02 Jan 2020)
+#
+# https://github.com/harisekhon/devop-python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+
+set -eu
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd "$srcdir/..";
+
+# shellcheck disable=SC1091
+. ./tests/utils.sh
+
+section "HexAnonymize"
+
+start_time=$(date +%s)
+
+run++
+check_output "abc123456def789012abcd" hexanonymize.py <<< "xyz987654rst654321AKIA"
+
+run++
+check_output "abc123456def789012ABCD" hexanonymize.py -c <<< "xyz987654rst654321AKIA"
+
+run++
+check_output "xyz123456rst789012abC" hexanonymize.py -c -o <<< "xyz987654rst654321caD"
+
+run++
+check_output "xyz123456rst789012abc" hexanonymize.py -o <<< "xyz987654rst654321caD"
+
+echo
+# $run_count defined in lib
+# shellcheck disable=SC2154
+echo "Total Tests run: $run_count"
+time_taken "$start_time" "All version tests for hexanonymize.py completed in"
+echo
+untrap
diff --git a/tests/test_json.sh b/tests/test_json.sh
index e7aafae63..34ab409d9 100755
--- a/tests/test_json.sh
+++ b/tests/test_json.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -25,6 +25,7 @@ echo "
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
until [ $# -lt 1 ]; do
@@ -35,9 +36,10 @@ done
# ignore multi-line json data file for spark testing
for jsonFile in $(find "${1:-.}" -iname '*.json' |
- grep -v '/spark-.*-bin-hadoop.*/' |
- grep -v 'multirecord.json' |
- grep -v -e 'broken' -e 'error'); do
+ grep -v -e '/spark-.*-bin-hadoop.*/' \
+ -e 'multirecord.json' \
+ -e 'broken' \
+ -e 'error'); do
echo "testing json file: $jsonFile"
python -mjson.tool < "$jsonFile" > /dev/null
done
diff --git a/tests/test_json_docs_to_bulk_multiline.sh b/tests/test_json_docs_to_bulk_multiline.sh
index 333688b75..53d08f35f 100755
--- a/tests/test_json_docs_to_bulk_multiline.sh
+++ b/tests/test_json_docs_to_bulk_multiline.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-07-30 14:30:00 +0200 (Sun, 30 Jul 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "Testing json_docs_to_bulk_multiline.py"
@@ -85,6 +86,7 @@ echo "testing stdin"
./json_docs_to_bulk_multiline.py - < "$data_dir/test.json" > "$stdout"
./json_docs_to_bulk_multiline.py < "$data_dir/test.json" > "$stdout"
echo "testing stdin and file mix"
+# shellcheck disable=SC2094
./json_docs_to_bulk_multiline.py "$data_dir/test.json" - < "$data_dir/test.json" > "$stdout"
# ==================================================
@@ -102,10 +104,11 @@ check_broken(){
filename="$1"
expected_exitcode="${2:-2}"
set +e
- ./json_docs_to_bulk_multiline.py "$filename" ${@:3} 2> "$stderr" > "$stdout"
+ # shellcheck disable=SC2086
+ ./json_docs_to_bulk_multiline.py "$filename" ${*:3} 2> "$stderr" > "$stdout"
exitcode=$?
set -e
- if [ $exitcode = $expected_exitcode ]; then
+ if [ $exitcode = "$expected_exitcode" ]; then
echo "successfully detected broken json in '$filename', returned exit code $exitcode"
echo
#elif [ $exitcode != 0 ]; then
diff --git a/tests/test_json_to_xml.sh b/tests/test_json_to_xml.sh
index 81a8f2669..8b66faa03 100755
--- a/tests/test_json_to_xml.sh
+++ b/tests/test_json_to_xml.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,20 +19,26 @@ srcdir="$(cd "$(dirname "$0")" && pwd)"
cd "$srcdir";
+# shellcheck disable=SC1091
. utils.sh
+
+# shellcheck disable=SC1091
. ../bash-tools/lib/utils.sh
section "JSON => XML"
-tmpfile="$(mktemp json_to_xml_test.XXXXX.xml)"
+#tmpfile="$(mktemp json_to_xml_test.XXXXX.xml)"
#echo "tmpfile is $tmpfile"
-trap "rm -f $tmpfile" $TRAP_SIGNALS
+#trap "rm -f $tmpfile" $TRAP_SIGNALS
+
+echo "running json_to_xml.py:"
+../json_to_xml.py data/test.json | tee /dev/stderr | validate_xml.py
+echo
-echo "running json_to_xml.py":
-../json_to_xml.py data/test.json | tee "$tmpfile"
+echo "running json_to_xml.py from stdin:"
+../json_to_xml.py < data/test.json | tee /dev/stderr | validate_xml.py
echo
-echo "now validating generated xml":
-../validate_xml.py "$tmpfile"
+echo "JSON to XML tests succeeded!"
echo
diff --git a/tests/test_json_to_yaml.sh b/tests/test_json_to_yaml.sh
new file mode 100755
index 000000000..4aa156043
--- /dev/null
+++ b/tests/test_json_to_yaml.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-12-19 18:04:15 +0000 (Thu, 19 Dec 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "$0")" && pwd)"
+
+cd "$srcdir/..";
+
+# shellcheck disable=SC1091,SC1090
+. "$srcdir/utils.sh"
+
+# shellcheck disable=SC1091,SC1090
+. "$srcdir/../bash-tools/lib/utils.sh"
+
+section "JSON => YAML"
+
+tmpfile="$(mktemp json_to_yaml_test.XXXXX.yml)"
+#echo "tmpfile is $tmpfile"
+
+# want var splitting
+# shellcheck disable=SC2086
+trap 'rm -f "$tmpfile"' $TRAP_SIGNALS
+
+for x in ./cloudformation/centos7-12nodes-encrypted.json tests/data/embedded_double_quotes.json; do
+ echo "running json_to_yaml.py $x"
+ ./json_to_yaml.py "$x" > "$tmpfile"
+ echo "now validating generated yaml"
+ ./validate_yaml.py "$tmpfile"
+ echo
+done
+
+echo "recursing directory to convert all json files under a directory tree to yaml"
+./json_to_yaml.py cloudformation/ > "$tmpfile"
+# TODO: fix validate_yaml.py to work on multi-yamls with --- and re-enable
+#echo "now validating generated yaml"
+#./validate_yaml.py "$tmpfile"
+echo "Success"
diff --git a/tests/test_opentsdb.sh b/tests/test_opentsdb.sh
index 70ceb7e3d..bbc59a11f 100755
--- a/tests/test_opentsdb.sh
+++ b/tests/test_opentsdb.sh
@@ -4,26 +4,24 @@
# Author: Hari Sekhon
# Date: 2016-10-10 11:54:19 +0100 (Mon, 10 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
+srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$srcdir/.."
-cd "$srcdir2/.."
-
-. "$srcdir2/utils.sh"
-
-srcdir="$srcdir2"
+# shellcheck disable=SC1090
+. "$srcdir/utils.sh"
echo "
# ============================================================================ #
@@ -41,7 +39,7 @@ export ZOOKEEPER_PORT=2181
export OPENTSDB_PORTS="$ZOOKEEPER_PORT $HBASE_STARGATE_PORT 8085 $HBASE_THRIFT_PORT 9095 3000 16000 16010 16201 16301"
export OPENTSDB_TEST_PORTS="$ZOOKEEPER_PORT $HBASE_THRIFT_PORT 3000"
-export OPENTSDB_VERSIONS="${@:-latest}"
+export OPENTSDB_VERSIONS="${*:-latest}"
#export DOCKER_IMAGE="opower/opentsdb"
#export DOCKER_IMAGE="petergrace/opentsdb-docker"
@@ -68,11 +66,13 @@ generate_test_data(){
#chars="$(echo {A..Z} {a..z} {0..9})"
chars=$(echo {A..Z} | tr -d ' ')
ts="$(date '+%s')"
+ # shellcheck disable=SC2034
for x in {1..100}; do
+ # shellcheck disable=SC2034
for y in {1..1000}; do
metric="metric${chars:$((RANDOM % ${#chars})):1}"
for z in {1..5}; do
- echo "ship${RANDOM:0:3} $(($ts + $RANDOM)) $RANDOM id=$metric crew=$z"
+ echo "ship${RANDOM:0:3} $((ts + RANDOM)) $RANDOM id=$metric crew=$z"
done
done
done > "$DATA_FILE"
@@ -111,6 +111,7 @@ made up error line
EOF
hr
echo "testing from data file and STDIN at the same time:"
+ # shellcheck disable=SC2094
./opentsdb_import_metric_distribution.py --key-prefix-length 7 "$DATA_FILE" - < "$DATA_FILE"
hr
@@ -119,7 +120,7 @@ EOF
}
for version in $OPENTSDB_VERSIONS; do
- test_opentsdb $version
+ test_opentsdb "$version"
done
if [ -z "${NODELETE:-}" ]; then
echo -n "removing test data: "
diff --git a/tests/test_presto.sh b/tests/test_presto.sh
index 7b47934f5..c031d5697 100755
--- a/tests/test_presto.sh
+++ b/tests/test_presto.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-09-22 17:01:38 +0200 (Fri, 22 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,12 +19,13 @@ srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "P r e s t o S Q L"
export PRESTO_TERADATA_VERSIONS="latest 0.152 0.157 0.167 0.179"
-export PRESTO_VERSIONS="${@:-${PRESTO_VERSIONS:-$PRESTO_TERADATA_VERSIONS}}"
+export PRESTO_VERSIONS="${*:-${PRESTO_VERSIONS:-$PRESTO_TERADATA_VERSIONS}}"
PRESTO_HOST="${DOCKER_HOST:-${PRESTO_HOST:-${HOST:-localhost}}}"
PRESTO_HOST="${PRESTO_HOST##*/}"
@@ -64,8 +65,8 @@ test_presto2(){
when_url_content "http://$PRESTO_HOST:$PRESTO_PORT/v1/service/presto/general" nodeId
hr
expected_version="$version"
- if [ "$version" = "latest" -o \
- "$version" = "NODOCKER" ]; then
+ if [ "$version" = "latest" ] ||
+ [ "$version" = "NODOCKER" ]; then
if [ "$teradata_distribution" = 1 ]; then
echo "latest version, fetching latest version from DockerHub master branch"
expected_version="$(dockerhub_latest_version presto)"
@@ -82,6 +83,7 @@ test_presto2(){
hr
PRESTO_PORT="$PRESTO_PORT_DEFAULT" ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_presto_coordinator.py $non_presto_node1 $non_presto_node2
+ # shellcheck disable=SC2097,SC2098
PRESTO_PORT="$PRESTO_PORT_DEFAULT" run_grep "^$PRESTO_HOST:$PRESTO_PORT$" ./find_active_presto_coordinator.py $non_presto_node1 "$PRESTO_HOST:$PRESTO_PORT"
echo "Completed $run_count Presto tests"
@@ -117,11 +119,12 @@ test_presto(){
fi
done
fi
- if [ "$teradata_distribution" = "1" -a $facebook_only -eq 0 ]; then
+ if [ "$teradata_distribution" = "1" ] &&
+ [ $facebook_only -eq 0 ]; then
echo "Testing Teradata's Presto distribution version: '$version'"
COMPOSE_FILE="$srcdir/docker/presto-docker-compose.yml" test_presto2 "$version"
# must call this manually here as we're sneaking in an extra batch of tests that run_test_versions is generally not aware of
- let total_run_count+=$run_count
+ ((total_run_count+=run_count))
# reset this so it can be used in test_presto to detect now testing Facebook
teradata_distribution=0
fi
diff --git a/tests/test_quay_show_tags.sh b/tests/test_quay_show_tags.sh
index c9c55ad62..96f451c23 100755
--- a/tests/test_quay_show_tags.sh
+++ b/tests/test_quay_show_tags.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/.."
+# shellcheck disable=SC1091
. "tests/utils.sh"
+# shellcheck disable=SC1091
. "bash-tools/lib/utils.sh"
section "Testing Quay.io Show Tags"
@@ -38,6 +40,8 @@ echo
echo
echo "All Quay Show Tags tests completed successfully"
echo
+# $run_count defined in lib
+# shellcheck disable=SC2154
echo "Total Tests run: $run_count"
time_taken "$start_time" "Quay Show Tags tests completed in"
echo
diff --git a/tests/test_serf_event_handler.sh b/tests/test_serf_event_handler.sh
index 00acd0fe7..d16c8e2f7 100755
--- a/tests/test_serf_event_handler.sh
+++ b/tests/test_serf_event_handler.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/.."
+# shellcheck disable=SC1091
. "tests/utils.sh"
+# shellcheck disable=SC1091
. "bash-tools/lib/utils.sh"
section "Testing Serf Event Handler"
diff --git a/tests/test_solrcloud.sh b/tests/test_solrcloud.sh
index bafb97a47..2f0f69196 100755
--- a/tests/test_solrcloud.sh
+++ b/tests/test_solrcloud.sh
@@ -4,28 +4,27 @@
# Author: Hari Sekhon
# Date: 2016-01-22 21:13:49 +0000 (Fri, 22 Jan 2016)
#
-# https://github.com/harisekhon/nagios-plugins
+# https://github.com/HariSekhon/Nagios-Plugins
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
-srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-cd "$srcdir2/.."
+cd "$srcdir/.."
+# shellcheck disable=SC1091
. ./tests/utils.sh
-srcdir="$srcdir2"
-
section "S o l r C l o u d"
-export SOLRCLOUD_VERSIONS="${@:-${SOLRCLOUD_VERSIONS:-latest 4.10 5.5 6.0 6.1 6.2 6.3 6.4 6.5 6.6}}"
+export SOLRCLOUD_VERSIONS="${*:-${SOLRCLOUD_VERSIONS:-latest 4.10 5.5 6.0 6.1 6.2 6.3 6.4 6.5 6.6}}"
SOLR_HOST="${DOCKER_HOST:-${SOLR_HOST:-${HOST:-localhost}}}"
SOLR_HOST="${SOLR_HOST##*/}"
@@ -46,11 +45,9 @@ trap_debug_env solr zookeeper
test_solrcloud(){
local version="$1"
# SolrCloud 4.x needs some different args / locations
- if [ ${version:0:1} = 4 ]; then
- four=true
+ if [ "${version:0:1}" = 4 ]; then
export SOLR_COLLECTION="collection1"
else
- four=""
export SOLR_COLLECTION="gettingstarted"
fi
section2 "Setting up SolrCloud $version docker test container"
@@ -63,7 +60,8 @@ test_solrcloud(){
hr
when_url_content "http://$SOLR_HOST:$SOLR_PORT/solr/" "Solr Admin"
hr
- local DOCKER_CONTAINER="$(docker-compose ps | sed -n '3s/ .*//p')"
+ local DOCKER_CONTAINER
+ DOCKER_CONTAINER="$(docker-compose ps | sed -n '3s/ .*//p')"
echo "container is $DOCKER_CONTAINER"
if [ -n "${NOTESTS:-}" ]; then
exit 0
@@ -73,6 +71,7 @@ test_solrcloud(){
hr
SOLR_PORT="$SOLR_PORT_DEFAULT" ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_solrcloud.py $non_solr_node1 $non_solr_node2
+ # shellcheck disable=SC2097,SC2098
SOLR_PORT="$SOLR_PORT_DEFAULT" run_grep "^$SOLR_HOST:$SOLR_PORT$" ./find_active_solrcloud.py $non_solr_node1 $non_solr_node2 "$SOLR_HOST:$SOLR_PORT"
docker-compose down
diff --git a/tests/test_spark_csv_to_avro.sh b/tests/test_spark_csv_to_avro.sh
index 05cf1bc17..5765ba160 100755
--- a/tests/test_spark_csv_to_avro.sh
+++ b/tests/test_spark_csv_to_avro.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir"
+# shellcheck disable=SC1091
. ./utils.sh
section "Spark CSV => Avro"
@@ -29,9 +30,9 @@ if is_inside_docker; then
exit 0
fi
-export SPARK_VERSIONS="${@:-1.4.0 1.5.1 1.6.2}"
+export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}"
# requires using spark-avro 3.0.0+
-#export SPARK_VERSIONS="${@:-2.0.0}"
+#export SPARK_VERSIONS="${*:-2.0.0}"
# don't support Spark <= 1.3 due to difference in databricks avro dependency
for SPARK_VERSION in $SPARK_VERSIONS; do
@@ -57,24 +58,36 @@ for SPARK_VERSION in $SPARK_VERSIONS; do
# resolved, was due to Spark 1.4+ requiring pyspark-shell for PYSPARK_SUBMIT_ARGS
rm -fr "test-header-$dir.avro"
- ../spark_csv_to_avro.py -c data/header.csv --has-header -a "test-header-$dir.avro" &&
- echo "SUCCEEDED with header with Spark $SPARK_VERSION" ||
- { echo "FAILED with header with Spark $SPARK_VERSION"; exit 1; }
+ if ../spark_csv_to_avro.py -c data/header.csv --has-header -a "test-header-$dir.avro"; then
+ echo "SUCCEEDED with header with Spark $SPARK_VERSION"
+ else
+ echo "FAILED with header with Spark $SPARK_VERSION"
+ exit 1
+ fi
rm -fr "test-header-schemaoverride-$dir.avro"
- ../spark_csv_to_avro.py -c data/header.csv -a "test-header-schemaoverride-$dir.avro" --has-header -s Year:String,Make,Model,Length:float &&
- echo "SUCCEEDED with header and schema override with Spark $SPARK_VERSION" ||
- { echo "FAILED with header and schema override with Spark $SPARK_VERSION"; exit 1; }
+ if ../spark_csv_to_avro.py -c data/header.csv -a "test-header-schemaoverride-$dir.avro" --has-header -s Year:String,Make,Model,Length:float; then
+ echo "SUCCEEDED with header and schema override with Spark $SPARK_VERSION"
+ else
+ echo "FAILED with header and schema override with Spark $SPARK_VERSION"
+ exit 1
+ fi
rm -fr "test-noheader-$dir.avro"
- ../spark_csv_to_avro.py -c data/test.csv -s Year:String,Make,Model,Length -a "test-noheader-$dir.avro" &&
- echo "SUCCEEDED with no header with Spark $SPARK_VERSION" ||
- { echo "FAILED with no header with Spark $SPARK_VERSION"; exit 1; }
+ if ../spark_csv_to_avro.py -c data/test.csv -s Year:String,Make,Model,Length -a "test-noheader-$dir.avro"; then
+ echo "SUCCEEDED with no header with Spark $SPARK_VERSION"
+ else
+ echo "FAILED with no header with Spark $SPARK_VERSION"
+ exit 1
+ fi
rm -fr "test-noheader-types-$dir.avro"
- ../spark_csv_to_avro.py -c data/test.csv -s Year:String,Make,Model,Length:float -a "test-noheader-types-$dir.avro" &&
- echo "SUCCEEDED with no header and float type with Spark $SPARK_VERSION" ||
- { echo "FAILED with no header and float type with Spark $SPARK_VERSION"; exit 1; }
+ if ../spark_csv_to_avro.py -c data/test.csv -s Year:String,Make,Model,Length:float -a "test-noheader-types-$dir.avro"; then
+ echo "SUCCEEDED with no header and float type with Spark $SPARK_VERSION"
+ else
+ echo "FAILED with no header and float type with Spark $SPARK_VERSION"
+ exit 1
+ fi
# if [ "$(cksum < "test-header-$dir.avro/part-r-00001.avro")" = "$(cksum < "test-noheader-$dir.avro/part-r-00001.avro")" ]; then
# echo "SUCCESSFULLY compared noheader with explicit schema and mixed implicit/explicit string types to headered csv avro output"
diff --git a/tests/test_spark_csv_to_parquet.sh b/tests/test_spark_csv_to_parquet.sh
index 1fc4b6b55..aa2b8d2dd 100755
--- a/tests/test_spark_csv_to_parquet.sh
+++ b/tests/test_spark_csv_to_parquet.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir"
+# shellcheck disable=SC1091
. ./utils.sh
section "Spark CSV => Parquet"
@@ -29,7 +30,7 @@ if is_inside_docker; then
exit 0
fi
-export SPARK_VERSIONS="${@:-1.3.1 1.4.0 1.5.1 1.6.2 2.0.0}"
+export SPARK_VERSIONS="${*:-1.3.1 1.4.0 1.5.1 1.6.2 2.0.0}"
for SPARK_VERSION in $SPARK_VERSIONS; do
dir="spark-$SPARK_VERSION-bin-hadoop2.6"
@@ -54,24 +55,36 @@ for SPARK_VERSION in $SPARK_VERSIONS; do
# resolved, was due to Spark 1.4+ requiring pyspark-shell for PYSPARK_SUBMIT_ARGS
rm -fr "test-header-$dir.parquet"
- ../spark_csv_to_parquet.py -c data/header.csv --has-header -p "test-header-$dir.parquet" &&
- echo "SUCCEEDED with header with Spark $SPARK_VERSION" ||
- { echo "FAILED with header with Spark $SPARK_VERSION"; exit 1; }
+ if ../spark_csv_to_parquet.py -c data/header.csv --has-header -p "test-header-$dir.parquet"; then
+ echo "SUCCEEDED with header with Spark $SPARK_VERSION"
+ else
+ echo "FAILED with header with Spark $SPARK_VERSION"
+ exit 1
+ fi
rm -fr "test-header-schemaoverride-$dir.parquet"
- ../spark_csv_to_parquet.py -c data/header.csv -p "test-header-schemaoverride-$dir.parquet" --has-header -s Year:String,Make,Model,Length:float &&
- echo "SUCCEEDED with header and schema override with Spark $SPARK_VERSION" ||
- { echo "FAILED with header and schema override with Spark $SPARK_VERSION"; exit 1; }
+ if ../spark_csv_to_parquet.py -c data/header.csv -p "test-header-schemaoverride-$dir.parquet" --has-header -s Year:String,Make,Model,Length:float; then
+ echo "SUCCEEDED with header and schema override with Spark $SPARK_VERSION"
+ else
+ echo "FAILED with header and schema override with Spark $SPARK_VERSION"
+ exit 1
+ fi
rm -fr "test-noheader-$dir.parquet"
- ../spark_csv_to_parquet.py -c data/test.csv -s Year:String,Make,Model,Length -p "test-noheader-$dir.parquet" &&
- echo "SUCCEEDED with no header with Spark $SPARK_VERSION" ||
- { echo "FAILED with no header with Spark $SPARK_VERSION"; exit 1; }
+ if ../spark_csv_to_parquet.py -c data/test.csv -s Year:String,Make,Model,Length -p "test-noheader-$dir.parquet"; then
+ echo "SUCCEEDED with no header with Spark $SPARK_VERSION"
+ else
+ echo "FAILED with no header with Spark $SPARK_VERSION"
+ exit 1
+ fi
rm -fr "test-noheader-types-$dir.parquet"
- ../spark_csv_to_parquet.py -c data/test.csv -s Year:String,Make,Model,Length:float -p "test-noheader-types-$dir.parquet" &&
- echo "SUCCEEDED with no header and float type with Spark $SPARK_VERSION" ||
- { echo "FAILED with no header and float type with Spark $SPARK_VERSION"; exit 1; }
+ if ../spark_csv_to_parquet.py -c data/test.csv -s Year:String,Make,Model,Length:float -p "test-noheader-types-$dir.parquet"; then
+ echo "SUCCEEDED with no header and float type with Spark $SPARK_VERSION"
+ else
+ echo "FAILED with no header and float type with Spark $SPARK_VERSION"
+ exit 1
+ fi
# if [ "$(cksum < "test-header-$dir.parquet/part-r-00001.parquet")" = "$(cksum < "test-noheader-$dir.parquet/part-r-00001.parquet")" ]; then
# echo "SUCCESSFULLY compared noheader with explicit schema and mixed implicit/explicit string types to headered csv parquet output"
diff --git a/tests/test_spark_json_to_avro.sh b/tests/test_spark_json_to_avro.sh
index 7c511eb3a..bb281026f 100755
--- a/tests/test_spark_json_to_avro.sh
+++ b/tests/test_spark_json_to_avro.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir"
+# shellcheck disable=SC1091
. ./utils.sh
section "Spark JSON => Avro"
@@ -30,9 +31,9 @@ if is_inside_docker; then
fi
# don't support Spark <= 1.3 due to difference in databricks avro dependency
-export SPARK_VERSIONS="${@:-1.4.0 1.5.1 1.6.2}"
+export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}"
# requires upgrade to spark-avro 3.0.0
-#export SPARK_VERSIONS="${@:-2.0.0}"
+#export SPARK_VERSIONS="${*:-2.0.0}"
for SPARK_VERSION in $SPARK_VERSIONS; do
dir="spark-$SPARK_VERSION-bin-hadoop2.6"
@@ -57,9 +58,12 @@ for SPARK_VERSION in $SPARK_VERSIONS; do
# resolved, was due to Spark 1.4+ requiring pyspark-shell for PYSPARK_SUBMIT_ARGS
rm -fr "test-$dir.avro"
- ../spark_json_to_avro.py -j data/multirecord.json -a "test-$dir.avro" &&
- echo "SUCCEEDED with header with Spark $SPARK_VERSION" ||
- { echo "FAILED with header with Spark $SPARK_VERSION"; exit 1; }
+ if ../spark_json_to_avro.py -j data/multirecord.json -a "test-$dir.avro"; then
+ echo "SUCCEEDED with header with Spark $SPARK_VERSION"
+ else
+ echo "FAILED with header with Spark $SPARK_VERSION"
+ exit 1
+ fi
#../spark_json_to_avro.py -j data/multirecord.json -a "test-$dir.avro" -s Year:String,Make,Model,Dimension.0.Length:float &&
# echo "SUCCEEDED with header with Spark $SPARK_VERSION" ||
diff --git a/tests/test_spark_json_to_parquet.sh b/tests/test_spark_json_to_parquet.sh
index 170631ba8..dc3e2fef4 100755
--- a/tests/test_spark_json_to_parquet.sh
+++ b/tests/test_spark_json_to_parquet.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir"
+# shellcheck disable=SC1091
. ./utils.sh
section "Spark JSON => Parquet"
@@ -29,7 +30,7 @@ if is_inside_docker; then
exit 0
fi
-export SPARK_VERSIONS="${@:-1.3.1 1.4.0 1.5.1 1.6.2 2.0.0}"
+export SPARK_VERSIONS="${*:-1.3.1 1.4.0 1.5.1 1.6.2 2.0.0}"
for SPARK_VERSION in $SPARK_VERSIONS; do
dir="spark-$SPARK_VERSION-bin-hadoop2.6"
@@ -50,8 +51,11 @@ for SPARK_VERSION in $SPARK_VERSIONS; do
echo
export SPARK_HOME="$dir"
rm -fr "test-$dir.parquet"
- ../spark_json_to_parquet.py -j data/multirecord.json -p "test-$dir.parquet" &&
- echo "SUCCEEDED with Spark $SPARK_VERSION" ||
- { echo "FAILED test with Spark $SPARK_VERSION"; exit 1; }
+ if ../spark_json_to_parquet.py -j data/multirecord.json -p "test-$dir.parquet"; then
+ echo "SUCCEEDED with Spark $SPARK_VERSION"
+ else
+ echo "FAILED test with Spark $SPARK_VERSION"
+ exit 1
+ fi
done
echo "SUCCESS"
diff --git a/tests/test_spark_z_avro_to_parquet.sh b/tests/test_spark_z_avro_to_parquet.sh
index ed20eb34a..b6c3ff7cb 100755
--- a/tests/test_spark_z_avro_to_parquet.sh
+++ b/tests/test_spark_z_avro_to_parquet.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir"
+# shellcheck disable=SC1091
. ./utils.sh
section "Spark Avro => Parquet"
@@ -29,9 +30,9 @@ if is_inside_docker; then
exit 0
fi
-export SPARK_VERSIONS="${@:-1.4.0 1.5.1 1.6.2}"
+export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}"
# requires using spark-avro 3.0.0+
-#export SPARK_VERSIONS="${@:-2.0.0}"
+#export SPARK_VERSIONS="${*:-2.0.0}"
for SPARK_VERSION in $SPARK_VERSIONS; do
dir="spark-$SPARK_VERSION-bin-hadoop2.6"
@@ -52,8 +53,11 @@ for SPARK_VERSION in $SPARK_VERSIONS; do
echo
export SPARK_HOME="$dir"
rm -fr "test-$dir.parquet"
- ../spark_avro_to_parquet.py -a "test-header-$dir.avro" -p "test-$dir.parquet" &&
- echo "SUCCEEDED with Spark $SPARK_VERSION" ||
- { echo "FAILED test with Spark $SPARK_VERSION"; exit 1; }
+ if ../spark_avro_to_parquet.py -a "test-header-$dir.avro" -p "test-$dir.parquet"; then
+ echo "SUCCEEDED with Spark $SPARK_VERSION"
+ else
+ echo "FAILED test with Spark $SPARK_VERSION"
+ exit 1
+ fi
done
echo "SUCCESS"
diff --git a/tests/test_spark_z_parquet_to_avro.sh b/tests/test_spark_z_parquet_to_avro.sh
index be4f764f8..9e02af7ed 100755
--- a/tests/test_spark_z_parquet_to_avro.sh
+++ b/tests/test_spark_z_parquet_to_avro.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir"
+# shellcheck disable=SC1091
. ./utils.sh
section "Spark Parquet => Avro"
@@ -29,9 +30,9 @@ if is_inside_docker; then
exit 0
fi
-export SPARK_VERSIONS="${@:-1.4.0 1.5.1 1.6.2}"
+export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}"
# requires using spark-avro 3.0.0+
-#export SPARK_VERSIONS="${@:-2.0.0}"
+#export SPARK_VERSIONS="${*:-2.0.0}"
for SPARK_VERSION in $SPARK_VERSIONS; do
dir="spark-$SPARK_VERSION-bin-hadoop2.6"
@@ -52,8 +53,11 @@ for SPARK_VERSION in $SPARK_VERSIONS; do
echo
export SPARK_HOME="$dir"
rm -fr "test-$dir.avro"
- ../spark_parquet_to_avro.py -p "test-$dir.parquet" -a "test-$dir.avro" &&
- echo "SUCCEEDED with Spark $SPARK_VERSION" ||
- { echo "FAILED test with Spark $SPARK_VERSION"; exit 1; }
+ if ../spark_parquet_to_avro.py -p "test-$dir.parquet" -a "test-$dir.avro"; then
+ echo "SUCCEEDED with Spark $SPARK_VERSION"
+ else
+ echo "FAILED test with Spark $SPARK_VERSION"
+ exit 1
+ fi
done
echo "SUCCESS"
diff --git a/tests/test_strip_ansi_escape_codes.sh b/tests/test_strip_ansi_escape_codes.sh
index b885c1878..d0effe274 100755
--- a/tests/test_strip_ansi_escape_codes.sh
+++ b/tests/test_strip_ansi_escape_codes.sh
@@ -15,6 +15,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "Strip ANSI Escape Codes"
@@ -23,11 +24,11 @@ name="strip_ansi_escape_codes.py"
start_time=$(date +%s)
-if is_mac; then
- cat_opts="-e"
-else
- cat_opts="-A"
-fi
+#if is_mac; then
+# cat_opts="-e"
+#else
+# cat_opts="-A"
+#fi
run++
if echo "some highlighted content" |
grep --color=yes highlighted |
@@ -42,6 +43,7 @@ fi
hr
tmp=$(mktemp /tmp/strip_ansi_escape_codes.XXXXX)
+# shellcheck disable=SC2064,SC2086
trap "rm $tmp" $TRAP_SIGNALS
echo
@@ -62,6 +64,8 @@ tee /dev/stderr |
fi
echo
+# $run_count defined in lib
+# shellcheck disable=SC2154
echo "Total Tests run: $run_count"
time_taken "$start_time" "All version tests for $name completed in"
echo
diff --git a/tests/test_timeout.sh b/tests/test_timeout.sh
index cadfde058..5d46e4bd6 100755
--- a/tests/test_timeout.sh
+++ b/tests/test_timeout.sh
@@ -4,18 +4,20 @@
# Author: Hari Sekhon
# Date: 2016-02-14 16:16:00 +0000 (Sun, 14 Feb 2016)
#
-# https://github.com/harisekhon/bash-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
-
srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-$srcdir/../timeout.py -t 2 sleep 10 || :
+# shellcheck disable=SC1090
+. "$srcdir/../bash-tools/lib/utils.sh"
+
+run_fail 3 "$srcdir/../timeout.py" -t 2 sleep 10
diff --git a/tests/test_validate_avro.sh b/tests/test_validate_avro.sh
index 7308e1ab5..b8bf5fbad 100755
--- a/tests/test_validate_avro.sh
+++ b/tests/test_validate_avro.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "Testing validate_avro.py"
@@ -26,8 +27,8 @@ section "Testing validate_avro.py"
export TIMEOUT=3
if [ $# -gt 0 ]; then
- echo "validate_avro.py $@"
- ./validate_avro.py $@
+ echo "validate_avro.py $*"
+ ./validate_avro.py "$@"
echo
fi
@@ -82,21 +83,23 @@ echo "testing stdin"
./validate_avro.py - < "$data_dir/test.avro"
./validate_avro.py < "$data_dir/test.avro"
echo "testing stdin mixed with filename"
+# shellcheck disable=SC2094
./validate_avro.py "$data_dir/test.avro" - < "$data_dir/test.avro"
echo
check_broken(){
local filename="$1"
local expected_exitcode="${2:-2}"
- local options="${@:3}"
+ local options="${*:3}"
set +e
+ # shellcheck disable=SC2086
./validate_avro.py -t 1 $options "$filename"
exitcode=$?
set -e
- if [ $exitcode = $expected_exitcode ]; then
+ if [ "$exitcode" = "$expected_exitcode" ]; then
echo "successfully detected broken avro in '$filename', returned exit code $exitcode"
echo
- #elif [ $exitcode != 0 ]; then
+ #elif [ "$exitcode" != 0 ]; then
# echo "returned unexpected non-zero exit code $exitcode for broken avro in '$filename'"
# exit 1
else
diff --git a/tests/test_validate_cson.sh b/tests/test_validate_cson.sh
index 20b9d332b..966a6d201 100755
--- a/tests/test_validate_cson.sh
+++ b/tests/test_validate_cson.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_validate_csv.sh b/tests/test_validate_csv.sh
index 18822a584..1747bebcc 100755
--- a/tests/test_validate_csv.sh
+++ b/tests/test_validate_csv.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "Testing validate_csv.py"
@@ -26,8 +27,8 @@ section "Testing validate_csv.py"
export TIMEOUT=3
if [ $# -gt 0 ]; then
- echo "validate_csv.py $@"
- ./validate_csv.py $@
+ echo "validate_csv.py $*"
+ ./validate_csv.py "$@"
echo
fi
@@ -70,6 +71,7 @@ echo "testing stdin"
./validate_csv.py - < "$data_dir/test.csv"
./validate_csv.py < "$data_dir/test.csv"
echo "testing stdin mixed with filename"
+# shellcheck disable=SC2094
./validate_csv.py "$data_dir/test.csv" - < "$data_dir/test.csv"
echo
@@ -83,12 +85,13 @@ hr2
check_broken(){
local filename="$1"
local expected_exitcode="${2:-2}"
- local options="${@:3}"
+ local options="${*:3}"
set +e
+ # shellcheck disable=SC2086
./validate_csv.py -t 1 $options "$filename"
exitcode=$?
set -e
- if [ $exitcode = $expected_exitcode ]; then
+ if [ "$exitcode" = "$expected_exitcode" ]; then
echo "successfully detected broken csv in '$filename', returned exit code $exitcode"
echo
#elif [ $exitcode != 0 ]; then
@@ -123,7 +126,7 @@ echo "checking blank content is invalid via stdin"
check_broken - 2 < "$broken_dir/blank.csv"
echo "checking blank content is invalid via stdin piped from /dev/null"
-cat /dev/null | check_broken - 2
+check_broken - 2 < /dev/null
echo
rm -fr "$broken_dir"
diff --git a/tests/test_validate_ini.sh b/tests/test_validate_ini.sh
index 32a658e9f..72349b24d 100755
--- a/tests/test_validate_ini.sh
+++ b/tests/test_validate_ini.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "Testing validate_ini.py"
@@ -26,8 +27,8 @@ section "Testing validate_ini.py"
export TIMEOUT=3
if [ $# -gt 0 ]; then
- echo "validate_ini.py $@"
- ./validate_ini.py $@
+ echo "validate_ini.py $*"
+ ./validate_ini.py "$@"
echo
fi
@@ -49,7 +50,8 @@ run_fail2(){
run_fail "${@/validate_ini/validate_ini2}" # ignore_run_unqualified
}
-if [ -f /etc/sssd/sssd.conf -a -r /etc/sssd/sssd.conf ]; then
+if [ -f /etc/sssd/sssd.conf ] &&
+ [ -r /etc/sssd/sssd.conf ]; then
run ./validate_ini.py /etc/sssd/sssd.conf
fi
@@ -86,6 +88,7 @@ echo "testing stdin"
run2 ./validate_ini.py - < "$data_dir/test.ini"
run2 ./validate_ini.py < "$data_dir/test.ini"
echo "testing stdin mixed with filename"
+# shellcheck disable=SC2094
run2 ./validate_ini.py "$data_dir/test.ini" - < "$data_dir/test.ini"
echo
@@ -110,12 +113,13 @@ export TIMEOUT=1
check_broken(){
local filename="$1"
local expected_exitcode="${2:-2}"
- local options="${@:3}"
+ local options="${*:3}"
set +e
+ # shellcheck disable=SC2086
./validate_ini.py $options "$filename"
exitcode=$?
set -e
- if [ $exitcode = $expected_exitcode ]; then
+ if [ "$exitcode" = "$expected_exitcode" ]; then
echo "successfully detected broken ini in '$filename', returned exit code $exitcode"
echo
#elif [ $exitcode != 0 ]; then
@@ -268,10 +272,10 @@ echo
hr2
echo "checking blank content is invalid via stdin piped from /dev/null"
-cat /dev/null | check_broken - 2
-cat /dev/null | run_fail 2 ./validate_ini.py
+check_broken - 2 < /dev/null
+run_fail 2 ./validate_ini.py < /dev/null
echo "validate_ini2.py blank content is valid:"
-cat /dev/null | run ./validate_ini2.py
+run ./validate_ini2.py < /dev/null
echo
hr2
diff --git a/tests/test_validate_json.sh b/tests/test_validate_json.sh
index fb600bae2..b92d16c2c 100755
--- a/tests/test_validate_json.sh
+++ b/tests/test_validate_json.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "Testing validate_json.py"
@@ -26,8 +27,8 @@ section "Testing validate_json.py"
export TIMEOUT=${TIMEOUT:-3}
if [ $# -gt 0 ]; then
- echo "validate_json.py $@"
- ./validate_json.py $@
+ echo "validate_json.py $*"
+ ./validate_json.py "$@"
echo
fi
@@ -68,6 +69,7 @@ echo "testing stdin"
./validate_json.py - < "$data_dir/test.json"
./validate_json.py < "$data_dir/test.json"
echo "testing stdin and file mix"
+# shellcheck disable=SC2094
./validate_json.py "$data_dir/test.json" - < "$data_dir/test.json"
echo "testing stdin with multirecord"
./validate_json.py -m - < "$data_dir/multirecord.json"
@@ -84,12 +86,13 @@ echo
check_broken(){
local filename="$1"
local expected_exitcode="${2:-2}"
- local options="${@:3}"
+ local options="${*:3}"
set +e
+ # shellcheck disable=SC2086
./validate_json.py $options "$filename"
exitcode=$?
set -e
- if [ $exitcode = $expected_exitcode ]; then
+ if [ "$exitcode" = "$expected_exitcode" ]; then
echo "successfully detected broken json in '$filename', returned exit code $exitcode"
echo
#elif [ $exitcode != 0 ]; then
@@ -210,17 +213,21 @@ echo "checking --permit-single-quotes mode infers multirecord single quoted json
echo
echo "checking --permit-single-quotes mode works with multirecord single quoted json with mixed quoting (should result in a WARNING message)"
-./validate_json.py -s "$data_dir/multirecord_single_double_mixed_quotes.notjson" -m 2>&1 |
- grep -q WARNING &&
- echo "Found warning message" ||
- { echo "failed to raise a WARNING message for mixed quoting"; exit 1; }
+if ./validate_json.py -s "$data_dir/multirecord_single_double_mixed_quotes.notjson" -m 2>&1 | grep -q WARNING; then
+ echo "Found warning message"
+else
+ echo "failed to raise a WARNING message for mixed quoting"
+ exit 1
+fi
echo
echo "checking --permit-single-quotes mode infers multirecord single quoted json with mixed quoting (should result in a WARNING message)"
-./validate_json.py -s "$data_dir/multirecord_single_double_mixed_quotes.notjson" 2>&1 |
- grep -q WARNING &&
- echo "Found warning message" ||
- { echo "failed to raise a WARNING message for mixed quoting"; exit 1; }
+if ./validate_json.py -s "$data_dir/multirecord_single_double_mixed_quotes.notjson" 2>&1 | grep -q WARNING; then
+ echo "Found warning message"
+else
+ echo "failed to raise a WARNING message while inferring mixed quoting"
+ exit 1
+fi
echo
# ============================================================================ #
@@ -309,7 +316,7 @@ echo "checking blank content is invalid for multirecord via stdin"
check_broken - 2 -m < "$broken_dir/blank.json"
echo "checking blank content is invalid for multirecord via stdin piped from /dev/null"
-cat /dev/null | check_broken - 2 -m
+check_broken - 2 -m < /dev/null
echo
check_broken_sample_files json
diff --git a/tests/test_validate_ldap_ldif.sh b/tests/test_validate_ldap_ldif.sh
index 566d90199..267269b32 100755
--- a/tests/test_validate_ldap_ldif.sh
+++ b/tests/test_validate_ldap_ldif.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-08-20 12:56:43 +0100 (Sun, 20 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,13 +19,14 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "Testing validate_ldap_ldif.py"
if [ $# -gt 0 ]; then
- echo "validate_ldap_ldif.py $@"
- ./validate_ldap_ldif.py $@
+ echo "validate_ldap_ldif.py $*"
+ ./validate_ldap_ldif.py "$@"
echo
fi
@@ -60,6 +61,7 @@ echo
echo "testing stdin"
./validate_ldap_ldif.py - < "$data_dir/add_ou.ldif"
./validate_ldap_ldif.py < "$data_dir/add_ou.ldif"
+# shellcheck disable=SC2094
./validate_ldap_ldif.py "$data_dir/add_ou.ldif" - < "$data_dir/add_ou.ldif"
echo
@@ -82,12 +84,13 @@ echo
check_broken(){
local filename="$1"
local expected_exitcode="${2:-2}"
- local options="${@:3}"
+ local options="${*:3}"
set +e
+ # shellcheck disable=SC2086
./validate_ldap_ldif.py $options "$filename"
exitcode=$?
set -e
- if [ $exitcode = $expected_exitcode ]; then
+ if [ "$exitcode" = "$expected_exitcode" ]; then
echo "successfully detected broken ldif in '$filename', returned exit code $exitcode"
echo
#elif [ $exitcode != 0 ]; then
diff --git a/tests/test_validate_multimedia.sh b/tests/test_validate_multimedia.sh
index 1efa5b4e6..3493bae6d 100755
--- a/tests/test_validate_multimedia.sh
+++ b/tests/test_validate_multimedia.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-05-01 20:46:42 +0100 (Sun, 01 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "Testing validate_multimedia.py"
@@ -31,15 +32,15 @@ if ! type -P ffmpeg &>/dev/null; then
echo "WARNING: ffmpeg not installed, skipping validate_multimedia.py tests"
exit 0
if type -P apt-get &>/dev/null; then
- sudo apt-get install -y ffmpeg
+ sudo apt-get install -o DPkg::Lock::Timeout=1200 -y ffmpeg
elif type -P yum &>/dev/null; then
echo "WARNING: cannot auto-install ffmpeg on RHEL/CentOS, the 3rd party repos and deps are seriously broken"
fi
fi
if [ $# -gt 0 ]; then
- echo "validate_multimedia.py $@"
- ./validate_multimedia.py $@
+ echo "validate_multimedia.py $*"
+ ./validate_multimedia.py "$@"
echo
fi
@@ -82,12 +83,13 @@ echo
check_broken(){
local filename="$1"
local expected_exitcode="${2:-2}"
- local options="${@:3}"
+ local options="${*:3}"
set +e
+ # shellcheck disable=SC2086
./validate_multimedia.py -t 1 $options "$filename"
exitcode=$?
set -e
- if [ $exitcode = $expected_exitcode ]; then
+ if [ "$exitcode" = "$expected_exitcode" ]; then
echo "successfully detected broken media in '$filename', returned exit code $exitcode"
echo
#elif [ $exitcode != 0 ]; then
diff --git a/tests/test_validate_parquet.sh b/tests/test_validate_parquet.sh
index 01d8a0e0f..44b0c8f7d 100755
--- a/tests/test_validate_parquet.sh
+++ b/tests/test_validate_parquet.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "Testing validate_parquet.py"
@@ -32,8 +33,8 @@ fi
#export TIMEOUT=10
if [ $# -gt 0 ]; then
- echo "validate_parquet.py $@"
- ./validate_parquet.py $@
+ echo "validate_parquet.py $*"
+ ./validate_parquet.py "$@"
echo
fi
@@ -84,18 +85,20 @@ echo "testing stdin"
./validate_parquet.py - < "$data_dir/test.parquet"
./validate_parquet.py < "$data_dir/test.parquet"
echo "testing stdin mixed with filename"
+# shellcheck disable=SC2094
./validate_parquet.py "$data_dir/test.parquet" - < "$data_dir/test.parquet"
echo
check_broken(){
local filename="$1"
local expected_exitcode="${2:-2}"
- local options="${@:3}"
+ local options="${*:3}"
set +e
+ # shellcheck disable=SC2086
./validate_parquet.py -t 5 $options "$filename"
exitcode=$?
set -e
- if [ $exitcode = $expected_exitcode ]; then
+ if [ "$exitcode" = "$expected_exitcode" ]; then
echo "successfully detected broken parquet in '$filename', returned exit code $exitcode"
echo
#elif [ $exitcode != 0 ]; then
diff --git a/tests/test_validate_toml.sh b/tests/test_validate_toml.sh
index 3dd37ed60..c20a38cf6 100755
--- a/tests/test_validate_toml.sh
+++ b/tests/test_validate_toml.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_validate_xml.sh b/tests/test_validate_xml.sh
index ae6479497..7fe631f3b 100755
--- a/tests/test_validate_xml.sh
+++ b/tests/test_validate_xml.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "Testing validate_xml.py"
@@ -26,8 +27,8 @@ section "Testing validate_xml.py"
export TIMEOUT=3
if [ $# -gt 0 ]; then
- echo "validate_xml.py $@"
- ./validate_xml.py $@
+ echo "validate_xml.py $*"
+ ./validate_xml.py "$@"
echo
fi
@@ -60,6 +61,7 @@ echo
echo "testing stdin"
./validate_xml.py - < "$data_dir/simple.xml"
./validate_xml.py < "$data_dir/simple.xml"
+# shellcheck disable=SC2094
./validate_xml.py "$data_dir/simple.xml" - < "$data_dir/simple.xml"
echo
@@ -72,12 +74,13 @@ echo "Now trying non-xml files to detect successful failure:"
check_broken(){
local filename="$1"
local expected_exitcode="${2:-2}"
- local options="${@:3}"
+ local options="${*:3}"
set +e
+ # shellcheck disable=SC2086
./validate_xml.py -t 1 $options "$filename"
exitcode=$?
set -e
- if [ $exitcode = $expected_exitcode ]; then
+ if [ "$exitcode" = "$expected_exitcode" ]; then
echo "successfully detected broken xml in '$filename', returned exit code $exitcode"
echo
#elif [ $exitcode != 0 ]; then
diff --git a/tests/test_validate_yaml.sh b/tests/test_validate_yaml.sh
index 7ab5b2975..abd4343ca 100755
--- a/tests/test_validate_yaml.sh
+++ b/tests/test_validate_yaml.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "Testing validate_yaml.py"
@@ -29,8 +30,8 @@ if is_inside_docker; then
fi
if [ $# -gt 0 ]; then
- echo "validate_yaml.py $@"
- ./validate_yaml.py $@
+ echo "validate_yaml.py $*"
+ ./validate_yaml.py "$@"
echo
fi
@@ -65,6 +66,7 @@ echo "testing stdin"
./validate_yaml.py - < "$data_dir/test.yaml"
./validate_yaml.py < "$data_dir/test.yaml"
echo "testing stdin mixed with filename"
+# shellcheck disable=SC2094
./validate_yaml.py "$data_dir/test.yaml" - < "$data_dir/test.yaml"
echo
@@ -77,12 +79,13 @@ echo "Now trying non-yaml files to detect successful failure:"
check_broken(){
local filename="$1"
local expected_exitcode="${2:-2}"
- local options="${@:3}"
+ local options="${*:3}"
set +e
+ # shellcheck disable=SC2086
./validate_yaml.py -t 1 $options "$filename"
exitcode=$?
set -e
- if [ $exitcode = $expected_exitcode ]; then
+ if [ "$exitcode" = "$expected_exitcode" ]; then
echo "successfully detected broken yaml in '$filename', returned exit code $exitcode"
echo
#elif [ $exitcode != 0 ]; then
diff --git a/tests/test_welcome.sh b/tests/test_welcome.sh
index 1be838108..0a13136a7 100755
--- a/tests/test_welcome.sh
+++ b/tests/test_welcome.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -20,6 +20,14 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
# shellcheck disable=SC1091
-#. ./tests/utils.sh
+. ./tests/utils.sh
+
+# Fedora doesn't have /var/log/wtmp
+if ! [ -f /var/log/wtmp ]; then
+ echo "/var/log/wtmp doesn't exist, touching..."
+ # assigned in utils.sh
+ # shellcheck disable=SC2154
+ $sudo touch /var/log/wtmp || :
+fi
./welcome.py "$@"
diff --git a/tests/test_xml_to_json.sh b/tests/test_xml_to_json.sh
index 17d15b2c7..6bd9cf731 100755
--- a/tests/test_xml_to_json.sh
+++ b/tests/test_xml_to_json.sh
@@ -4,35 +4,43 @@
# Author: Hari Sekhon
# Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
-srcdir="$(cd "$(dirname "$0")" && pwd)"
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-cd "$srcdir";
+cd "$srcdir"
-. utils.sh
-. ../bash-tools/lib/utils.sh
+# shellcheck disable=SC1091,SC1090
+. "$srcdir/utils.sh"
+
+# shellcheck disable=SC1091,SC1090
+. "$srcdir/../bash-tools/lib/utils.sh"
section "XML => JSON"
-for x in simple.xml plant_catalog.xml; do
- tmpfile="$(mktemp xml_to_json_test.XXXXX.xml)"
- trap "rm -f $tmpfile" $TRAP_SIGNALS
- echo "running xml_to_json.py on $x":
- ../xml_to_json.py "data/$x" > "$tmpfile"
- echo
- echo "now validating generated json":
- ../validate_json.py "$tmpfile"
- echo
- rm -f "$tmpfile"
- echo "========="
-done
+cd "$srcdir/.."
+
+testdata="tests/data/simple.xml"
+
+echo "running xml_to_json.py on $testdata":
+./xml_to_json.py "$testdata" | tee /dev/stderr | ./validate_json.py
+echo
+
+echo "running xml_to_json.py on stdin < $testdata":
+./xml_to_json.py < "$testdata" | tee /dev/stderr | ./validate_json.py
+echo
+
+echo "running xml_to_json.py on tests/data/plant_catalog.xml":
+./xml_to_json.py "tests/data/plant_catalog.xml" | ./validate_json.py
+echo
+echo "XML to JSON tests succeeded!"
+echo
diff --git a/tests/test_xml_to_yaml.sh b/tests/test_xml_to_yaml.sh
new file mode 100755
index 000000000..4b3d72da0
--- /dev/null
+++ b/tests/test_xml_to_yaml.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+cd "$srcdir"
+
+# shellcheck disable=SC1091,SC1090
+. "$srcdir/utils.sh"
+
+# shellcheck disable=SC1091,SC1090
+. "$srcdir/../bash-tools/lib/utils.sh"
+
+section "XML => YAML"
+
+cd "$srcdir/.."
+
+testdata="tests/data/simple.xml"
+
+echo "running xml_to_yaml.py on $testdata":
+./xml_to_yaml.py "$testdata" | tee /dev/stderr | ./validate_yaml.py
+echo
+
+echo "running xml_to_yaml.py on stdin < $testdata":
+./xml_to_yaml.py < "$testdata" | tee /dev/stderr | ./validate_yaml.py
+echo
+
+echo "running xml_to_yaml.py on tests/data/plant_catalog.xml":
+./xml_to_yaml.py "tests/data/plant_catalog.xml" | ./validate_yaml.py
+echo
+echo "XML to yaml tests succeeded!"
+echo
diff --git a/tests/test_yamllint.sh b/tests/test_yamllint.sh
index 31d808ff6..90c983ce4 100755
--- a/tests/test_yamllint.sh
+++ b/tests/test_yamllint.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2019-02-26 14:40:53 +0000 (Tue, 26 Feb 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/utils.sh b/tests/utils.sh
index 57c32ca46..eddb33d01 100755
--- a/tests/utils.sh
+++ b/tests/utils.sh
@@ -5,13 +5,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/timeout.py b/timeout.py
index b274872a3..831d2e99f 100755
--- a/timeout.py
+++ b/timeout.py
@@ -1,16 +1,16 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-02-14 15:46:37 +0000 (Sun, 14 Feb 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # pylint: disable=line-too-long
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/travis_debug_session.py b/travis_debug_session.py
index 6812eeead..aafebf35e 100755
--- a/travis_debug_session.py
+++ b/travis_debug_session.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-08-10 18:18:03 +0100 (Wed, 10 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -48,6 +48,7 @@
import sys
import time
import traceback
+import git
try:
import requests
except ImportError:
@@ -68,7 +69,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.8.4'
+__version__ = '0.9.1'
class TravisDebugSession(CLI):
@@ -151,10 +152,24 @@ def process_options(self):
self.repo = travis_user + self.repo
validate_chars(self.repo, 'repo', r'\/\w\.-')
else:
- self.usage('--job-id / --repo not specified')
- validate_alnum(self.travis_token, 'travis token')
+ self.repo = self.get_local_repo_name()
+ if not self.repo:
+ self.usage('--job-id / --repo not specified')
+ validate_alnum(self.travis_token, 'travis token', is_secret=True)
self.headers['Authorization'] = 'token {0}'.format(self.travis_token)
+ @staticmethod
+ def get_local_repo_name():
+ try:
+ _ = git.Repo('.')
+ for remote in _.remotes:
+ for url in remote.urls:
+ repo = '/'.join(url.split('/')[-2:])
+ log.debug('determined repo to be {} from remotes'.format(repo))
+ return repo
+ except git.InvalidGitRepositoryError:
+ log.debug('failed to determine git repository locally: %s', _)
+
def run(self):
if not self.job_id:
if self.repo:
diff --git a/travis_last_log.py b/travis_last_log.py
index b9b6843f2..82b7493e5 100755
--- a/travis_last_log.py
+++ b/travis_last_log.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-08-10 18:18:03 +0100 (Wed, 10 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -61,6 +61,7 @@
#import string
import sys
import traceback
+import git
srcdir = os.path.abspath(os.path.dirname(__file__))
libdir = os.path.join(srcdir, 'pylib')
sys.path.append(libdir)
@@ -76,7 +77,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.5.0'
+__version__ = '0.6.1'
class TravisLastBuildLog(CLI):
@@ -147,8 +148,10 @@ def process_options(self):
self.repo = travis_user + self.repo
validate_chars(self.repo, 'repo', r'\/\w\.-')
else:
- self.usage('--job-id / --repo not specified')
- validate_alnum(self.travis_token, 'travis token')
+ self.repo = self.get_local_repo_name()
+ if not self.repo:
+ self.usage('--job-id / --repo not specified')
+ validate_alnum(self.travis_token, 'travis token', is_secret=True)
self.headers['Authorization'] = 'token {0}'.format(self.travis_token)
self.num = self.get_opt('num')
validate_int(self.num, 'num', 1)
@@ -163,6 +166,18 @@ def process_options(self):
#if not self.color and not (sys.__stdin__.isatty() and sys.__stdout__.isatty()):
# self.plaintext = True
+ @staticmethod
+ def get_local_repo_name():
+ try:
+ _ = git.Repo('.')
+ for remote in _.remotes:
+ for url in remote.urls:
+ repo = '/'.join(url.split('/')[-2:])
+ log.debug('determined repo to be {} from remotes'.format(repo))
+ return repo
+ except git.InvalidGitRepositoryError:
+ log.debug('failed to determine git repository locally: %s', _)
+
def run(self):
if self.job_id:
self.print_log(job_id=self.job_id)
diff --git a/urldecode.py b/urldecode.py
new file mode 100755
index 000000000..87319d3be
--- /dev/null
+++ b/urldecode.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-03-03 17:34:06 +0000 (Tue, 03 Mar 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tool to url decode text from standard input or a text argument
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import os
+import sys
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon import CLI
+ from harisekhon.utils import isPythonMinVersion
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+# pylint: disable=no-name-in-module,import-error
+if isPythonMinVersion(3):
+ from urllib.parse import unquote_plus as unquote
+else:
+ from urllib import unquote_plus as unquote
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.1'
+
+
+class URLDecode(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(URLDecode, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.timeout_default = None
+
+ def run(self):
+ if len(sys.argv) > 1:
+ for arg in sys.argv[1:]:
+ self.decode(arg)
+ else:
+ # buffered - Control-D char meshes with late output
+ #for line in sys.stdin:
+ while True:
+ line = sys.stdin.readline()
+ if not line:
+ break
+ line = line.rstrip('\n').rstrip('\r')
+ self.decode(line)
+
+ @staticmethod
+ def decode(string):
+ print(unquote(string))
+ sys.stdout.flush()
+
+
+if __name__ == '__main__':
+ URLDecode().main()
diff --git a/urlencode.py b/urlencode.py
new file mode 100755
index 000000000..6ca907f15
--- /dev/null
+++ b/urlencode.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-03-03 17:34:06 +0000 (Tue, 03 Mar 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tool to url encode text from standard input or a text argument
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import os
+import sys
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon import CLI
+ from harisekhon.utils import isPythonMinVersion
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+# pylint: disable=no-name-in-module,import-error
+if isPythonMinVersion(3):
+ from urllib.parse import quote_plus as quote
+else:
+ from urllib import quote_plus as quote
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.1'
+
+
+class URLEncode(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(URLEncode, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.timeout_default = None
+
+ def run(self):
+ if len(sys.argv) > 1:
+ for arg in sys.argv[1:]:
+ self.encode(arg)
+ else:
+ # buffered - Control-D char meshes with late output
+ #for line in sys.stdin:
+ while True:
+ line = sys.stdin.readline()
+ if not line:
+ break
+ line = line.rstrip('\n').rstrip('\r')
+ self.encode(line)
+
+ @staticmethod
+ def encode(string):
+ #print(urllib.parse.quote(string))
+ print(quote(string))
+ sys.stdout.flush()
+
+
+if __name__ == '__main__':
+ URLEncode().main()
diff --git a/validate_avro.py b/validate_avro.py
index f8a896466..37eaab848 100755
--- a/validate_avro.py
+++ b/validate_avro.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -39,7 +39,7 @@
try:
from avro3.datafile import DataFileReader, DataFileException
from avro3.io import DatumReader
-except:
+except Exception: # pylint: disable=broad-except
from avro.datafile import DataFileReader, DataFileException
from avro.io import DatumReader
libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
@@ -55,7 +55,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.9.1'
+__version__ = '0.9.2'
class AvroValidatorTool(CLI):
diff --git a/validate_cson.py b/validate_cson.py
index 4287f6d93..6e017ab97 100755
--- a/validate_cson.py
+++ b/validate_cson.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-10-07 10:23:31 +0100 (Mon, 07 Oct 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_csv.py b/validate_csv.py
index c53d16eb1..bac1d583e 100755
--- a/validate_csv.py
+++ b/validate_csv.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_ini.py b/validate_ini.py
index 84411046d..b38b3ab6a 100755
--- a/validate_ini.py
+++ b/validate_ini.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2017-09-15 15:29:39 +0200 (Fri, 15 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -58,7 +58,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.12.2'
+__version__ = '0.12.3'
class IniValidatorTool(CLI):
@@ -72,7 +72,7 @@ def __init__(self):
self.re_suffix = re.compile(r'.*\.(?:ini|properties)$', re.I)
# In Windows ini key cannot contain equals sign = or semicolon ;
# key=val or [section]
- self.re_ini_section = re.compile(r'^\s*\[([\w=\:\.-]+)\]\s*$')
+ self.re_ini_section = re.compile(r'^\s*\[([\w\s=\:\.-]+)\]\s*$')
self.re_ini_key = re.compile(r'^\s*(?:[^\[;=]+)s*$')
# INI value can be anything .* so not regex'ing it
self.valid_ini_msg = ' => INI OK'
diff --git a/validate_ini2.py b/validate_ini2.py
index aaf17f87f..5c41436a4 100755
--- a/validate_ini2.py
+++ b/validate_ini2.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2017-09-15 15:29:39 +0200 (Fri, 15 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_json.py b/validate_json.py
index aab399e41..dbca2c431 100755
--- a/validate_json.py
+++ b/validate_json.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -58,7 +58,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.11.0'
+__version__ = '0.11.2'
class JsonValidatorTool(CLI):
@@ -210,7 +210,7 @@ def check_json(self, content):
return True
self.failed = True
if not self.passthru:
- die(self.self.invalid_json_msg_single_quotes)
+ die(self.invalid_json_msg_single_quotes)
else:
log.debug('not valid json')
if self.rewind_check_multirecord_json():
@@ -297,8 +297,6 @@ def check(self, filename):
if filename == '-':
filename = ''
self.filename = filename
- self.valid_json_msg = '{0} => JSON OK'.format(filename)
- self.invalid_json_msg = '{0} => JSON INVALID'.format(filename)
single_quotes = '(found single quotes not double quotes)'
self.valid_json_msg_single_quotes = '{0} {1}'.format(self.valid_json_msg, single_quotes)
self.invalid_json_msg_single_quotes = '{0} {1}'.format(self.invalid_json_msg, single_quotes)
diff --git a/validate_ldap_ldif.py b/validate_ldap_ldif.py
index 02e3e4bf7..963b725c8 100755
--- a/validate_ldap_ldif.py
+++ b/validate_ldap_ldif.py
@@ -1,18 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# coding=utf-8
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2017-08-04 00:53:10 +0200 (Fri, 04 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_multimedia.py b/validate_multimedia.py
index 9452fd6c7..0362bb30d 100755
--- a/validate_multimedia.py
+++ b/validate_multimedia.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-01 20:46:56 +0100 (Sun, 01 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -45,7 +45,7 @@
import re
import sys
import subprocess
-from subprocess import CalledProcessError
+CalledProcessError = subprocess.CalledProcessError
libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
sys.path.append(libdir)
try:
@@ -164,8 +164,8 @@ def check_path(self, path):
die("failed to determine if path '%s' is file or directory" % path)
def check_media_file(self, filename):
- if self.is_excluded(filename):
- return
+ #if self.is_excluded(filename):
+ # return
valid_media_msg = '%s => OK' % filename
invalid_media_msg = '%s => INVALID' % filename
cmd = self.validate_cmd
diff --git a/validate_parquet.py b/validate_parquet.py
index 71894645f..bf6690d06 100755
--- a/validate_parquet.py
+++ b/validate_parquet.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_toml.py b/validate_toml.py
index b75c2dce0..25628d222 100755
--- a/validate_toml.py
+++ b/validate_toml.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-10-07 10:23:31 +0100 (Mon, 07 Oct 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_xml.py b/validate_xml.py
index 5620a81e5..c954bac42 100755
--- a/validate_xml.py
+++ b/validate_xml.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_yaml.py b/validate_yaml.py
index 2852a1905..75ee4034c 100755
--- a/validate_yaml.py
+++ b/validate_yaml.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -49,7 +49,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.9.1'
+__version__ = '0.9.3'
class YamlValidatorTool(CLI):
@@ -86,7 +86,7 @@ def is_excluded(self, path):
return False
def check_yaml(self, content):
- if isYaml(content):
+ if isYaml(content, safe_load_all=True):
if self.get_opt('print'):
print(content, end='')
else:
@@ -99,7 +99,7 @@ def check_yaml(self, content):
if not self.get_opt('print'):
if self.verbose > 2:
try:
- yaml.safe_load(content)
+ yaml.safe_load_all(content)
except yaml.YAMLError as _:
print(_)
die(self.invalid_yaml_msg)
diff --git a/welcome.py b/welcome.py
index 97f6f173f..d9c1b2546 100755
--- a/welcome.py
+++ b/welcome.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2009-12-09 19:58:14 +0000 (Wed, 09 Dec 2009)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -45,7 +45,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '2.0.1'
+__version__ = '2.0.4'
class Welcome(CLI):
@@ -56,9 +56,10 @@ def __init__(self):
# Python 3.x
# super().__init__()
self.quick = False
+ self.timeout_default = 20
@staticmethod
- def case_user(user):
+ def titlecase_user(user):
if user == 'root':
user = user.upper()
elif len(user) < 4 or re.search(r'\d', user):
@@ -75,7 +76,7 @@ def construct_msg(self):
# print("invalid user '%s' determined from environment variable $USER, failed regex validation" % user)
print("invalid user '%s' returned by getpass.getuser(), failed regex validation" % user)
sys.exit(ERRORS['CRITICAL'])
- user = self.case_user(user)
+ user = self.titlecase_user(user)
msg = 'Welcome %s - ' % user
last = ''
if which("last"):
@@ -120,7 +121,7 @@ def print_welcome(self):
print(msg)
return
try:
- charmap = list(string.uppercase + string.lowercase + '@#$%^&*()')
+ charmap = list(string.ascii_uppercase + string.ascii_lowercase + '@#$%^&*()')
# print '',
# print('', end='')
for i in range(0, len(msg)):
diff --git a/xml_to_json.py b/xml_to_json.py
index 10484bfce..14d4cde62 100755
--- a/xml_to_json.py
+++ b/xml_to_json.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-01-15 00:07:09 +0000 (Fri, 15 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# http://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -51,7 +51,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.1'
+__version__ = '0.2.0'
class XmlToJson(CLI):
@@ -116,7 +116,7 @@ def process_file(self, filepath):
if filepath == '-':
filepath = ''
if filepath == '':
- self.xml_to_json(sys.stdin.read())
+ print(self.xml_to_json(sys.stdin.read()))
else:
with open(filepath) as _:
content = _.read()
diff --git a/xml_to_yaml.py b/xml_to_yaml.py
new file mode 100755
index 000000000..47a098243
--- /dev/null
+++ b/xml_to_yaml.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-12-19 18:19:34 +0000 (Thu, 19 Dec 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tool to convert XML to YAML
+
+Reads any given files as XML and prints the equivalent YAML to stdout for piping or redirecting to a file.
+
+Directories if given are detected and recursed, processing all files in the directory tree ending in a .xml suffix.
+
+Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from
+standard input.
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import json
+import os
+import re
+import sys
+import xml
+import xmltodict
+import yaml
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import die, ERRORS, log, log_option
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+
+class XmlToYaml(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(XmlToYaml, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.indent = None
+ self.re_xml_suffix = re.compile(r'.*\.xml$', re.I)
+
+ def add_options(self):
+ self.add_opt('-p', '--pretty', action='store_true', help='Pretty Print the resulting YAML')
+
+ @staticmethod
+ def xml_to_yaml(content, filepath=None):
+ try:
+ _ = xmltodict.parse(content)
+ except xml.parsers.expat.ExpatError as _:
+ file_detail = ''
+ if filepath is not None:
+ file_detail = ' in file \'{0}\''.format(filepath)
+ die("Failed to parse XML{0}: {1}".format(file_detail, _))
+ # xmltodict returns a unicode OrderedDict so need to make it a plain dict to come out properly not like:
+ # !!python/object/apply:collections.OrderedDict
+ yaml_string = yaml.safe_dump(json.loads(json.dumps(_)), encoding='utf-8', sort_keys=True)
+ return yaml_string
+
+ def run(self):
+ if self.get_opt('pretty'):
+ log_option('pretty', True)
+ self.indent = 4
+ if not self.args:
+ self.args.append('-')
+ for arg in self.args:
+ if arg == '-':
+ continue
+ if not os.path.exists(arg):
+ print("'{}' not found".format(arg))
+ sys.exit(ERRORS['WARNING'])
+ if os.path.isfile(arg):
+ log_option('file', arg)
+ elif os.path.isdir(arg):
+ log_option('directory', arg)
+ else:
+ die("path '{}' could not be determined as either a file or directory".format(arg))
+ for arg in self.args:
+ self.process_path(arg)
+
+ def process_path(self, path):
+ if path == '-' or os.path.isfile(path):
+ self.process_file(path)
+ elif os.path.isdir(path):
+ for root, _, files in os.walk(path):
+ for filename in files:
+ filepath = os.path.join(root, filename)
+ if self.re_xml_suffix.match(filepath):
+ self.process_file(filepath)
+ else:
+ die("failed to determine if path '{}' is a file or directory".format(path))
+
+ def process_file(self, filepath):
+ log.debug("processing filepath '%s'", filepath)
+ if filepath == '-':
+ filepath = ''
+ if filepath == '':
+ print(self.xml_to_yaml(sys.stdin.read()))
+ else:
+ with open(filepath) as _:
+ content = _.read()
+ print(self.xml_to_yaml(content, filepath=filepath))
+
+
+if __name__ == '__main__':
+ XmlToYaml().main()
diff --git a/yaml_to_json.py b/yaml_to_json.py
new file mode 100755
index 000000000..90a5db9b7
--- /dev/null
+++ b/yaml_to_json.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+# args: ../bash-tools/.gitlab-ci.yml
+#
+# Author: Hari Sekhon
+# Date: 2020-08-18 01:17:56 +0100 (Tue, 18 Aug 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tool to convert YAML to JSON
+
+Reads any given files as YAML and prints the equivalent JSON to stdout for piping or redirecting to a file.
+
+Directories if given are detected and recursed, processing all files in the directory tree ending in a
+.yml / .yaml suffix.
+
+Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from
+standard input.
+
+Written to convert .gitlab-ci.yml files to JSON for inputting to the GitLab API for validation
+(see gitlab_validate_ci_yml.sh in the DevOps-Bash-tools repo)
+
+See also:
+
+ yaml2json.sh - https://github.com/HariSekhon/DevOps-Bash-tools
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import json
+import os
+import re
+import sys
+import yaml
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import die, ERRORS, log, log_option
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.2.0'
+
+
+class YamlToJson(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(YamlToJson, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.re_yaml_suffix = re.compile(r'.*\.ya?ml$', re.I)
+
+ @staticmethod
+ def yaml_to_json(content, filepath=None):
+ try:
+ _ = yaml.load(content, Loader=yaml.FullLoader)
+ except (KeyError, ValueError) as _:
+ file_detail = ''
+ if filepath is not None:
+ file_detail = ' in file \'{0}\''.format(filepath)
+ die("Failed to parse YAML{0}: {1}".format(file_detail, _))
+ return json.dumps(_, sys.stdout, indent=4)
+
+ def run(self):
+ if not self.args:
+ self.args.append('-')
+ for arg in self.args:
+ if arg == '-':
+ continue
+ if not os.path.exists(arg):
+ print("'%s' not found" % arg)
+ sys.exit(ERRORS['WARNING'])
+ if os.path.isfile(arg):
+ log_option('file', arg)
+ elif os.path.isdir(arg):
+ log_option('directory', arg)
+ else:
+ die("path '%s' could not be determined as either a file or directory" % arg)
+ for arg in self.args:
+ self.process_path(arg)
+
+ def process_path(self, path):
+ if path == '-' or os.path.isfile(path):
+ self.process_file(path)
+ elif os.path.isdir(path):
+ for root, _, files in os.walk(path):
+ for filename in files:
+ filepath = os.path.join(root, filename)
+ if self.re_yaml_suffix.match(filepath):
+ self.process_file(filepath)
+ else:
+ die("failed to determine if path '%s' is a file or directory" % path)
+
+ def process_file(self, filepath):
+ log.debug('processing filepath \'%s\'', filepath)
+ if filepath == '-':
+ filepath = ''
+ if filepath == '':
+ print(self.yaml_to_json(sys.stdin.read()))
+ else:
+ with open(filepath) as _:
+ content = _.read()
+ print(self.yaml_to_json(content, filepath=filepath))
+
+
+if __name__ == '__main__':
+ YamlToJson().main()